Running as root is not permitted on build environment

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

.env

@@ -1,4 +1,4 @@
 REPO=milvusdb/milvus-distributed-dev
 ARCH=amd64
 UBUNTU=18.04
-SHOW_DATE=none
+DATE_VERSION=20201102-104943

.github/workflows/main.yaml

@@ -44,15 +44,6 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           reporter: github-pr-check # Default is github-pr-check
           hadolint_ignore: DL3008
-      - name: Docker Pull
-        shell: bash
-        run: |
-          docker-compose pull --ignore-pull-failures ubuntu
-      - name: Docker Build
-        shell: bash
-        run: |
-          docker-compose build ubuntu
-          docker rmi $(docker images | grep '<none>' | awk '{print $3}') || exit 0
       - name: Cache Docker Volumes
         uses: actions/cache@v1
         with:
@@ -63,14 +54,6 @@ jobs:
         shell: bash
         run: |
           cd ${GITHUB_WORKSPACE}/deployments/docker && docker-compose up -d
-      - name: Docker Run
+      - name: Build and UnitTest
         run: |
-          docker-compose run ubuntu
-      - name: Docker Push
-        if: success() && github.event_name == 'push' && github.repository == 'zilliztech/milvus-distributed'
-        continue-on-error: true
-        shell: bash
-        run: |
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker-compose push ubuntu
+          ./build/builder.sh

.github/workflows/publish-builder.yaml

@@ -40,7 +40,7 @@ jobs:
       - name: Docker Build
         shell: bash
         run: |
-          SHOW_DATE=${{ steps.extracter.outputs.version }} docker-compose build ubuntu
+          DATE_VERSION=${{ steps.extracter.outputs.version }} docker-compose build ubuntu
       - name: Docker Push
         if: success() && github.event_name == 'push' && github.repository == 'zilliztech/milvus-distributed'
         continue-on-error: true
@@ -48,4 +48,4 @@ jobs:
         run: |
           docker login -u ${{ secrets.DOCKERHUB_USER }} \
                        -p ${{ secrets.DOCKERHUB_TOKEN }}
-          SHOW_DATE=${{ steps.extracter.outputs.version }} docker-compose push ubuntu
+          DATE_VERSION=${{ steps.extracter.outputs.version }} docker-compose push ubuntu

Makefile

@@ -36,9 +36,9 @@ fmt:
 lint:
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=1m --config ./.golangci.yml ./internal/...
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=1m --config ./.golangci.yml ./cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=1m --config ./.golangci.yml ./test/...
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=3m --config ./.golangci.yml ./internal/...
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=3m --config ./.golangci.yml ./cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=3m --config ./.golangci.yml ./test/...
 
 ruleguard:
 	@echo "Running $@ check"

build/builder.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Absolute path to the toplevel milvus distributed directory.
+toplevel=$(dirname "$(cd "$(dirname "${0}")"; pwd)")
+
+pushd "${toplevel}"
+
+if [ "${1-}" = "pull" ]; then
+    docker-compose pull --ignore-pull-failures ubuntu
+    exit 0
+fi
+
+if [ "${1-}" = "gdbserver" ]; then
+	mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-ubuntu18.04-gdbserver-cache"
+	chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}"
+    docker-compose up -d gdbserver
+    exit 0
+fi
+
+# Attempt to run in the container with the same UID/GID as we have on the host,
+# as this results in the correct permissions on files created in the shared
+# volumes. This isn't always possible, however, as IDs less than 100 are
+# reserved by Debian, and IDs in the low 100s are dynamically assigned to
+# various system users and groups. To be safe, if we see a UID/GID less than
+# 500, promote it to 501. This is notably necessary on macOS Lion and later,
+# where administrator accounts are created with a GID of 20. This solution is
+# not foolproof, but it works well in practice.
+uid=$(id -u)
+gid=$(id -g)
+[ "$uid" -lt 500 ] && uid=501
+[ "$gid" -lt 500 ] && gid=$uid
+
+mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-ubuntu18.04-cache"
+chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}"
+
+docker-compose pull --ignore-pull-failures ubuntu
+docker-compose build ubuntu
+docker-compose run --rm -u "$uid:$gid" ubuntu "$@"
+
+popd

build/docker/env/cpu/ubuntu18.04/Dockerfile

@@ -38,13 +38,31 @@ ENV GOROOT /usr/local/go
 ENV GO111MODULE on
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
 RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.15.2.linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \
-    mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" && \
-    go get github.com/golang/protobuf/protoc-gen-go@v1.3.2
+    mkdir -p "$GOPATH/src" "$GOPATH/bin" && \
+    go get github.com/golang/protobuf/protoc-gen-go@v1.3.2 && chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR)
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    openssh-server gdb gdbserver && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Taken from - https://docs.docker.com/engine/examples/running_ssh_service/#environment-variables
+
+RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
+    useradd -u 2000 -ms /bin/bash debugger && echo 'debugger:milvus' | chpasswd
+
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+ENV NOTVISIBLE "in users profile"
+RUN echo "export VISIBLE=now" >> /etc/profile
+
+# 22 for ssh server. 7777 for gdb server.
+EXPOSE 22 7777
 
-# Set permissions on /etc/passwd and /home to allow arbitrary users to write
 COPY --chown=0:0 build/docker/env/entrypoint.sh /
-RUN mkdir -p /home/user && chgrp -R 0 /home && chmod -R g=u /etc/passwd /etc/group /home && chmod +x /entrypoint.sh
 
-ENV HOME=/home/user
-ENTRYPOINT [ "/entrypoint.sh" ]
+RUN wget -qO- "https://github.com/benesch/autouseradd/releases/download/1.2.0/autouseradd-1.2.0-amd64.tar.gz" | tar xz -C / --strip-components 1
+
+ENTRYPOINT [ "autouseradd", "--user", "milvus", "--", "/entrypoint.sh" ]
 CMD ["tail", "-f", "/dev/null"]

docker-compose.yml

@@ -9,13 +9,13 @@ x-ccache: &ccache
 
 services:
   ubuntu:
-    image: ${REPO}:${ARCH}-ubuntu${UBUNTU}-${SHOW_DATE}
+    image: ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION}
     network_mode: "host"
     build:
       context: .
       dockerfile: build/docker/env/cpu/ubuntu${UBUNTU}/Dockerfile
       cache_from:
-        - ${REPO}:${ARCH}-ubuntu${UBUNTU}-${SHOW_DATE}
+        - ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION}
     shm_size: 2G
     environment:
       <<: *ccache
@@ -26,3 +26,23 @@ services:
     command: &ubuntu-command >
       /bin/bash -c "
         make build-cpp && make verifiers && make unittest"
+
+  gdbserver:
+    image: ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION}
+    build:
+      context: .
+      dockerfile: build/docker/env/cpu/ubuntu${UBUNTU}/Dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION}
+    security_opt: # options needed for gdb debugging
+      - seccomp:unconfined
+      - apparmor:unconfined
+    environment:
+      <<: *ccache
+    volumes:
+      - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${ARCH}-ubuntu${UBUNTU}-gdbserver-cache:/ccache:delegated
+    container_name: debugger
+    entrypoint: ["/usr/sbin/sshd", "-D"]
+    ports:
+      - "7776:22"
+      - "7777:7777"

internal/core/CMakeLists.txt

@@ -190,8 +190,6 @@ if ( BUILD_UNIT_TEST STREQUAL "ON" )
     add_subdirectory(unittest)
 endif ()
 
-
-
 add_custom_target( Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean )
 
 # **************************** Install ****************************