Enhanced testing and remove run as root (#108)

* adding runAsNonRoot and init containers to set permissions * updating to quotes and typo in security context * adding specific user ids * updating to use runAsUserId in service * updating for platformsvcs to run as non-root * adding logging to tests
microsoft · Aug 5, 2024 · bc7733c · bc7733c
1 parent dd6a88a
commit bc7733c
Show file tree

Hide file tree

Showing 9 changed files with 324 additions and 8 deletions.
diff --git a/chart/templates/_fileserver.tpl b/chart/templates/_fileserver.tpl
@@ -140,6 +140,20 @@ mountPath: {{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeNam
 {{- end }}
 {{- end }}
 
+{{- define "spacefx.fileserver.clientapp.volumemount.mountpath" }}
+{{- $serviceValues := .serviceValues }}
+{{- $globalValues := .globalValues }}
+{{- $volumeName := .volumeName }}
+{{- $shareName := printf "%s-%s" $serviceValues.appName $volumeName }}
+{{- $mountPath := printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }}
+{{- if and (eq $serviceValues.appName "hostsvc-link") (eq $volumeName "allxfer") }}
+{{ printf "%s/%s" $globalValues.spacefxDirectories.base $volumeName }}
+{{- else }}
+{{ printf "%s/%s/%s" $globalValues.spacefxDirectories.base $volumeName $serviceValues.appName }}
+{{- end }}
+{{- end }}
+
+
 {{- define "spacefx.fileserver.clientapp.volume" }}
 {{- $serviceValues := .serviceValues }}
 {{- $volumeName := .volumeName }}

diff --git a/chart/templates/_initcontainers.tpl b/chart/templates/_initcontainers.tpl
@@ -0,0 +1,25 @@
+{{- define "spacefx.initcontainers.setperms" }}
+{{- $serviceValues := .serviceValues }}
+{{- $globalValues := .globalValues }}
+{{- $mountPaths := "" }}
+initContainers:
+  - name: init-permissions
+    image: docker.io/rancher/mirrored-library-busybox:1.36.1
+    volumeMounts:
+{{- range $volumeKey, $volumeName := $globalValues.xferVolumes }}
+{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }}
+{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }}
+{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }}
+{{- printf "- %s" $fileServerVolumeMount | nindent 5 }}
+{{- end }}
+{{- if $serviceValues.xferVolumes }}
+{{- range $volumeKey, $volumeName := $serviceValues.xferVolumes }}
+{{- $fileServerVolumeMount := printf "%s" (include "spacefx.fileserver.clientapp.volumemount" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | nindent 2 | trim) }}
+{{- $fileServerVolumeMountPath := printf "%s" (include "spacefx.fileserver.clientapp.volumemount.mountpath" (dict "globalValues" $globalValues "serviceValues" $serviceValues "volumeName" $volumeName) | trim) }}
+{{- $mountPaths = printf "%s %s" $mountPaths $fileServerVolumeMountPath }}
+{{- printf "- %s" $fileServerVolumeMount | nindent 5 }}
+{{- end }}
+{{- end }}
+    command: ["sh", "-c", "chown -R {{ $serviceValues.runAsUserId }}:{{ $serviceValues.runAsUserId }} {{ $mountPaths }}"]
+{{- end }}
+
diff --git a/chart/templates/_service.tpl b/chart/templates/_service.tpl
@@ -44,9 +44,18 @@ spec:
       {{- if eq $serviceValues.appName "platform-mts" }}
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
+      {{- end }}
+      {{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }}
+{{- include "spacefx.initcontainers.setperms" (dict "globalValues" $globalValues "serviceValues" $serviceValues) | indent 6 }}
       {{- end }}
       containers:
         - name: {{ $serviceValues.appName | quote }}
+          {{- if and ($globalValues.security.forceNonRoot) (ne $serviceValues.appName "hostsvc-link") }}
+          securityContext:
+            runAsUser: {{ $serviceValues.runAsUserId }}
+            runAsGroup: {{ $serviceValues.runAsUserId }}
+            runAsNonRoot: true
+          {{- end }}
           ports:
           - name: app-port
             containerPort: 50051
@@ -70,12 +79,6 @@ spec:
             timeoutSeconds: {{ $globalValues.probes.startup.timeoutSeconds }}
           {{- end }}
         {{- end }}
-        {{- if $serviceValues.securityContext }}
-          securityContext:
-        {{- range $index, $securityContext := $serviceValues.securityContext }}
-            {{ $securityContext.name }}: {{ $securityContext.value }}
-        {{- end }}
-        {{- end }}
         {{- if $serviceValues.debugShim }}
           image: "{{ $serviceValues.repository }}:latest"
           imagePullPolicy: "Never"

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -113,6 +113,7 @@ global:
   security:
     payloadAppNetworkRestrictionsEnabled: true
     topicRestrictionEnabled: true
+    forceNonRoot: true
 subcharts:
   dapr:
     enabled: false
@@ -134,6 +135,7 @@ services:
         workingDirectory: /workspaces/exampleapp
       repository: kaniko-project/executor
       tag: v1.20.1-slim
+      runAsUserId: 701
     registry:
       appName: coresvc-registry
       containerCommand:
@@ -163,6 +165,7 @@ services:
         cpu:
           limit: 1000m
           request: 10m
+      runAsUserId: 702
     switchboard:
       appName: coresvc-switchboard
       serviceNamespace: coresvc
@@ -192,6 +195,7 @@ services:
         enabled: true
         hasBase: false
       enabled: false
+      runAsUserId: 703
   platform:
     mts:
       appConfig:
@@ -215,6 +219,7 @@ services:
         hasBase: false
       enabled: false
       workingDir: /workspaces/platform-mts
+      runAsUserId: 704
     deployment:
       appName: platform-deployment
       appHealthChecks: true
@@ -269,6 +274,7 @@ services:
         hasBase: false
       enabled: false
       workingDir: /workspace/platform-deployment
+      runAsUserId: 705
     vth:
       appConfig:
         - name: enableRoutingToMTS
@@ -291,6 +297,7 @@ services:
         hasBase: false
       enabled: false
       workingDir: /workspaces/vth
+      runAsUserId: 706
   host:
     sensor:
       appConfig:
@@ -315,6 +322,7 @@ services:
         enabled: true
         hasBase: false
       enabled: false
+      runAsUserId: 707
       workingDir: /workspaces/hostsvc-sensor
     link:
       appConfig:
@@ -371,6 +379,7 @@ services:
         enabled: true
         hasBase: false
       enabled: false
+      runAsUserId: 708
       workingDir: /workspaces/hostsvc-logging
     position:
       appName: hostsvc-position
@@ -390,6 +399,7 @@ services:
         enabled: true
         hasBase: false
       enabled: false
+      runAsUserId: 709
       workingDir: /workspaces/hostsvc-position
   payloadapp:
     payloadappTemplate:

diff --git a/scripts/deploy_spacefx.sh b/scripts/deploy_spacefx.sh
@@ -86,11 +86,15 @@ function deploy_spacefx_service_group(){
     info_log "Scanning '${service_group}' spacefx services for deploying..."
 
     run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group} | to_entries[] | select(.value.${enabled_filter} == true) | .key' -r" spacefx_services
+    run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot
+
     run_a_script "kubectl --kubeconfig ${KUBECONFIG} get deployments -A -o json" services_deployed_cache --disable_log
 
     for service in $spacefx_services; do
         run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.appName' -r" spacefx_service_appName
         run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.serviceNamespace' -r" spacefx_service_serviceNamespace
+        run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.services.${service_group}.${service}.runAsUserId' -r" spacefx_service_userid
+
 
         run_a_script "jq -r '.items[] | select(.metadata.name == \"${spacefx_service_appName}\" and (.metadata.namespace == \"${spacefx_service_serviceNamespace}\")) | true' <<< \${services_deployed_cache}" service_deployed
 
@@ -99,6 +103,78 @@ function deploy_spacefx_service_group(){
             continue
         fi
 
+        # Create users and groups if the service needs one
+        if [[ "${spacefx_forceNonRoot}" == "true" ]] && [[ "${spacefx_service_userid}" != "null" ]]; then
+            info_log "...checking if group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}') exists..."
+
+            # This will return the group_name for the groupID.  i.e. "702"
+            run_a_script "getent group ${spacefx_service_userid}" preexisting_groupid_by_id --ignore_error
+            preexisting_groupid_by_id="${preexisting_groupid_by_id%%:*}"
+
+            # This will check if a group exists and gets its ID
+            run_a_script "getent group ${spacefx_service_appName}" preexisting_groupid_by_name --ignore_error
+            preexisting_groupid_by_name="${preexisting_groupid_by_name%%:*}"
+
+
+            if [[ -n "${preexisting_groupid_by_name}" ]] && [[ "${preexisting_groupid_by_id}" == "${preexisting_groupid_by_name}" ]]; then
+                info_log "...group '${spacefx_service_appName}' (GID: '${preexisting_groupid_by_id}') already exists.  Nothing to do"
+            else
+                if [[ -n "${preexisting_groupid_by_id}" ]]; then
+                    info "...GID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'.  Attempting to delete..."
+                    run_a_script "getent group ${spacefx_service_userid}" group_to_del
+                    group_to_del="${group_to_del%%:*}"
+
+                    run_a_script "groupdel -f ${group_to_del}"
+                    info "...successfully deleted previous group '${group_to_del}' (GID: '${username_to_del}')"
+                fi
+
+                if [[ -n "${preexisting_groupid_by_name}" ]]; then
+                    info "...Group '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'.  Attempting to delete..."
+                    run_a_script "groupdel -f ${spacefx_service_appName}"
+                    info "...successfully deleted previous group '${spacefx_service_appName}'"
+                fi
+
+                info_log "...creating group '${spacefx_service_appName}' with GID '${spacefx_service_userid}'..."
+                run_a_script "groupadd -r -g ${spacefx_service_userid} ${spacefx_service_appName}" --no_log
+                info_log "...successfully created group '${spacefx_service_appName}' (GID: '${spacefx_service_userid}')."
+            fi
+
+
+            info_log "...checking if user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') exists..."
+
+            # This will return a user id if the userid exists.  i.e. "701"
+            run_a_script "id -u ${spacefx_service_userid}" preexisting_userid --ignore_error
+
+            # This will return the user id for the username.  i.e. "702"
+            run_a_script "id -u ${spacefx_service_appName}" preexisting_userid_for_username --ignore_error
+
+            if [[ -n "${preexisting_userid_for_username}" ]] && [[ "${preexisting_userid}" == "${preexisting_userid_for_username}" ]]; then
+                info_log "...user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}') already exists.  Nothing to do"
+            else
+                if [[ -n "${preexisting_userid}" ]]; then
+                    info "...UID '${spacefx_service_userid}' already in use, but isn't assigned to '${spacefx_service_appName}'.  Attempting to delete..."
+                    run_a_script "getent passwd ${spacefx_service_userid}" username_to_del
+                    username_to_del="${username_to_del%%:*}"
+                    run_a_script "userdel -f ${username_to_del}"
+                    info "...successfully deleted previous user '${username_to_del}' (UID: '${username_to_del}')"
+                fi
+
+                if [[ -n "${preexisting_userid_for_username}" ]]; then
+                    info "...Username '${spacefx_service_appName}' already in use, but isn't assigned to '${spacefx_service_userid}'.  Attempting to delete..."
+                    run_a_script "userdel -f ${spacefx_service_appName}"
+                    info "...successfully deleted previous user '${spacefx_service_appName}' (UID: '${preexisting_userid_for_username}')"
+                fi
+
+                info_log "...creating user '${spacefx_service_appName}' with UID '${spacefx_service_userid}'..."
+                run_a_script "useradd -r -u ${spacefx_service_userid} -g ${spacefx_service_appName} -d /nonexistent -s /usr/sbin/nologin ${spacefx_service_appName}" --no_log
+                info_log "...successfully created user '${spacefx_service_appName}' (UID: '${spacefx_service_userid}')."
+            fi
+
+
+        fi
+
+
+
         info_log "...adding '${service}'..."
         deploy_group_cmd="${deploy_group_cmd} --set services.${service_group}.${service}.enabled=true \
                                                 --set services.${service_group}.${service}.provisionVolumeClaims=true \
@@ -203,6 +279,15 @@ function deploy_apps_to_deployment_service(){
     info_log "...successfully copied chart to '${SPACEFX_DIR}/xfer/platform-deployment/tmp/chart/${SPACEFX_VERSION}'"
 
 
+    run_a_script "yq '.' ${SPACEFX_DIR}/chart/values.yaml --output-format=json | jq '.global.security.forceNonRoot' -r" spacefx_forceNonRoot
+
+    if [[ "${spacefx_forceNonRoot}" == "true" ]]; then
+        info_log "Updating permissions for '${SPACEFX_DIR}/xfer/platform-deployment' to user 'platform-deployment'..."
+        run_a_script "chown -R platform-deployment:platform-deployment ${SPACEFX_DIR}/xfer/platform-deployment"
+        run_a_script "chmod -R u+rwx ${SPACEFX_DIR}/xfer/platform-deployment"
+        info_log "Permissions successfully updated"
+    fi
+
     info_log "FINISHED: ${FUNCNAME[0]}"
 }
 

diff --git a/scripts/stage_spacefx.sh b/scripts/stage_spacefx.sh
@@ -239,6 +239,10 @@ function toggle_security_restrictions(){
         run_a_script "yq eval '.global.security.topicRestrictionEnabled = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
         info_log "'DEV_ENVIRONMENT' = true.  Allowing Links to Platform-Deployment..."
         run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = true' -i \"${SPACEFX_DIR}/chart/values.yaml\""
+
+        info_log "'DEV_ENVIRONMENT' = true.  Enabling run as root..."
+        run_a_script "yq eval '.global.security.forceNonRoot = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
+
         info_log "...security restrictions configured for development"
     else
         info_log "'DEV_ENVIRONMENT' = false.  Enabling Network Restrictions..."
@@ -251,6 +255,9 @@ function toggle_security_restrictions(){
         run_a_script "yq eval '(.services.host.link.appConfig[] | select(.name == \"allowLinksToDeploymentSvc\") .value) = false' -i \"${SPACEFX_DIR}/chart/values.yaml\""
         info_log "...Network and Topic restrictions successfully enabled."
 
+        info_log "'DEV_ENVIRONMENT' = true.  Enabling run as root..."
+        run_a_script "yq eval '.global.security.forceNonRoot = true' -i \"${SPACEFX_DIR}/chart/values.yaml\""
+
         info_log "...security restrictions configured for production"
     fi
 

diff --git a/tests/dev_cluster.sh b/tests/dev_cluster.sh
@@ -7,6 +7,7 @@
 #
 #  "bash ./tests/dev_cluster.sh"
 set -e
+MAX_WAIT_SECS=300
 SCRIPT_NAME=$(basename "$0")
 WORKING_DIR="$(git rev-parse --show-toplevel)"
 
@@ -22,6 +23,57 @@ ARTIFACT_PATH=${WORKING_DIR}/output/spacefx-dev/devcontainer-feature-spacefx-dev
 
 echo "Microsoft Azure Orbital Space SDK - Development Cluster Test"
 
+############################################################
+# Given a namespace, this function will wait for all pods to enter a running state
+############################################################
+function wait_for_namespace_to_provision(){
+
+    local namespace=""
+
+    while [[ "$#" -gt 0 ]]; do
+        case $1 in
+            --namespace)
+                shift
+                namespace=$1
+                ;;
+        esac
+        shift
+    done
+
+    if [[ -z $namespace ]]; then
+        echo "--namespace not provided to wait_for_namespace_to_provision function"
+        exit 1
+    fi
+
+    echo "Waiting for namespace '${namespace}' to fully provision (max $MAX_WAIT_SECS seconds)..."
+
+    # This returns any pods that are not completed nor succeeded
+    k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length')
+
+    start_time=$(date +%s)
+
+    while [[ $k3s_deployments_not_ready != "0" ]]; do
+        k3s_deployments_not_ready=$(kubectl get deployments --kubeconfig "${KUBECONFIG}" -n "${namespace}" --output=json | jq '[.items[] | select(.spec.replicas != .status.availableReplicas)] | length')
+
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+
+        if [[ $elapsed_time -ge $MAX_WAIT_SECS ]]; then
+            echo "Timed out waiting for deployment to complete.  Check logs for more information"
+            exit 1
+        fi
+
+        kubectl get pods -A
+        kubectl get deployments -A
+
+        echo "Found incomplete deployments.  Rechecking in 5 seconds"
+        sleep 5
+    done
+
+    echo "Namespace '${namespace}' is provisioned"
+
+}
+
 if [[ -d "/var/spacedev" ]]; then
     echo "Preexisting /var/spacedev found.  Resetting enviornment with big_red_button.sh"
     /var/spacedev/scripts/big_red_button.sh
@@ -94,6 +146,9 @@ if [[ ! -f "${KUBECONFIG}" ]]; then
     echo "KUBECONFIG '${KUBECONFIG}' not found.  Cluster did not initialize."
     exit 1
 fi
+
+
+
 kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-registry -n coresvc
 kubectl --kubeconfig ${KUBECONFIG} get deployment/coresvc-switchboard -n coresvc
 
@@ -107,6 +162,9 @@ kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-deployment -n platfor
 kubectl --kubeconfig ${KUBECONFIG} get deployment/platform-mts -n platformsvc
 kubectl --kubeconfig ${KUBECONFIG} get deployment/vth -n platformsvc
 
+wait_for_namespace_to_provision --namespace coresvc
+wait_for_namespace_to_provision --namespace hostsvc
+wait_for_namespace_to_provision --namespace platformsvc
 
 echo ""
 echo ""