Merge pull request #1 from mthreads/mingyuanw/ci

[MTAI-489] build(ci): test CI

Jenkinsfile

@@ -0,0 +1,117 @@
+class PodTemplateFiles {
+  private Map files = [
+    'MThreads GPU': 'ci/templates/musa.yaml',
+  ]
+
+  public String getPodTemplateFile(String platform) {
+    String file = files.get(platform)
+    return file
+  }
+}
+
+def ifTriggeredByTimer() {
+  return currentBuild.getBuildCauses()[0].shortDescription == 'Started by timer'
+}
+
+pipeline {
+  parameters {
+    choice(name: 'HARDWARE_PLATFORM', choices: ['MThreads GPU'], description: 'Target hardware platform')
+  }
+
+  agent {
+    kubernetes {
+      yamlFile "${new PodTemplateFiles().getPodTemplateFile(params.HARDWARE_PLATFORM)}"
+      defaultContainer "main"
+    }
+  }
+
+  triggers {
+    // UTC
+    cron(env.BRANCH_NAME == 'develop' ? '0 18 * * *' : '')
+  }
+
+  environment {
+    paddle_musa_working_dir = '/paddle_musa'
+  }
+
+  stages {
+    stage("CI-Clone") {
+      // Move PR contents to the specific directory, mainly to make use of the pre-built artifacts(saving time for compiling)
+      steps {
+        container('main') {
+          sh """#!/bin/bash
+          mkdir -p ${env.paddle_musa_working_dir}
+          cp -r ${env.WORKSPACE}/. ${env.paddle_musa_working_dir}
+          """
+        }
+      }
+    }
+    stage("Codestyle-Check") {
+      steps {
+        // when using `dir()` directive, always raise java.nio.file.AccessDeniedException,
+        // so change working directory within the shell scripts
+        container('main') {
+            sh """#!/bin/bash
+            cd ${env.paddle_musa_working_dir}
+            git config --global --add safe.directory "*"
+            BRANCH=origin/develop /bin/bash tools/codestyle/pre_commit.sh
+            #git diff --name-only origin/develop..HEAD | xargs pre-commit run --files
+            """
+        }
+      }
+    }
+    stage('Build') {
+      steps {
+        container('main') {
+          sh """#!/bin/bash
+          cd ${env.paddle_musa_working_dir}
+          /bin/bash ci/build.sh -j64
+          """
+        }
+      }
+    }
+    stage('Unit Test') {
+      steps {
+        container('main') {
+          sh 'echo Unit Test'
+          sh """#!/bin/bash
+          cd ${env.paddle_musa_working_dir}/build
+          ctest -V -R system_allocator_test
+          """
+          // sh '/bin/bash --login -c "cd build && ctest -V -R system_allocator_test"'
+        }
+      }
+    }
+    stage('Integration Test') {
+      steps {
+        container('main') {
+          sh 'echo Integration Test'
+        }
+      }
+    }
+    stage('Daily Release') {
+      agent {
+        kubernetes {
+          yamlFile 'ci/templates/musa.yaml'
+          defaultContainer "main"
+        }
+      }
+      when {
+        beforeAgent true
+        allOf {
+          branch 'develop'
+          expression { ifTriggeredByTimer() }
+        }
+      }
+      steps {
+        container('main') {
+          sh 'echo Daily Release in main'
+
+        }
+        container('release') {
+          sh 'echo Daily Release in release'
+        }
+      }
+    }
+  }
+}

ci/build.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+BUILD_FROM_SCRATCH=0
+MAX_JOBS=8
+BUILD_ARTIFACTS_URL="https://oss.mthreads.com/mt-ai-data/paddle_musa/pre_build/paddle_musa_build_artifacts.tar.gz"
+CCACHE_ARTIFACTS_URL="https://oss.mthreads.com/mt-ai-data/paddle_musa/pre_build/paddle_musa_ccache_artifacts.tar.gz"
+
+CUR_DIR=$(cd "$(dirname "$0")"; pwd)
+PADDLE_MUSA_DIR=$(dirname "$CUR_DIR")
+
+usage() {
+  echo -e "\033[1;32mThis script is used to build paddle_musa on CI. \033[0m"
+  echo -e "\033[1;32mParameters usage: \033[0m"
+  echo -e "\033[32m    --from_scratch    : Means building paddle_musa from scratch. \033[0m"
+  echo -e "\033[32m    -j/--jobs         : Means number of threads used for compiling. \033[0m"
+  echo -e "\033[32m    -h/--help         : Help information. \033[0m"
+}
+
+parameters=`getopt -o j:h:: --long jobs:,from_scratch,help::, -n "$0" -- "$@"`
+[ $? -ne 0 ] && { echo -e "\033[34mTry '$0 --help' for more information. \033[0m"; exit 1; }
+
+eval set -- "$parameters"
+
+while true;do
+    case "$1" in
+        --from_scratch) BUILD_FROM_SCRATCH=1; shift ;;
+        -j|--jobs) MAX_JOBS=$2 ; shift 2;;
+        -h|--help) usage;exit ;;
+        --)
+            shift ; break ;;
+        *) usage;exit 1;;
+    esac
+done
+
+pushd ${PADDLE_MUSA_DIR}
+# prepare submodules by copying from the local repo,
+# in this case, CI docker need to be updated once submodules' version changed
+cp -r ${PADDLE_MUSA_REPO_PATH}/third_party/. third_party
+cp -r ${PADDLE_MUSA_REPO_PATH}/.git/modules .git
+
+export INFERENCE_DEMO_INSTALL_DIR="/home/data/paddle_musa/.cache/build"
+export CCACHE_DIR="/home/data/paddle_musa/.ccache/build"
+
+
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+if [ "${CURRENT_BRANCH}" = "develop" ]; then
+  ref_revision=$(git log -2 --pretty=%H | tail -1)
+else
+  ref_revision="origin/develop"
+fi
+
+cmake_diff_nums=$(git diff --name-only ${ref_revision} | grep "CMake\|cmake" | wc -l)
+if [ ${cmake_diff_nums} -gt 0 ]; then
+  BUILD_FROM_SCRATCH=1
+fi
+
+ccache -s
+
+if [ ${BUILD_FROM_SCRATCH} -eq 0 ]; then
+  # prepare the pre-built artifacts
+  wget -q --no-check-certificate ${CCACHE_ARTIFACTS_URL} -O ./ccache_artifacts.tar.gz
+  wget -q --no-check-certificate ${BUILD_ARTIFACTS_URL} -O ./build_artifacts.tar.gz
+
+  tar zmxf build_artifacts.tar.gz
+  tar zmxf ccache_artifacts.tar.gz -C ${CCACHE_DIR}
+  rm ccache_artifacts.tar.gz build_artifacts.tar.gz
+  rm build/build_summary.txt
+fi
+
+git diff --name-only ${ref_revision} | xargs touch
+
+find build -name "CMakeCache*" | xargs rm
+export MAX_JOBS=${MAX_JOBS}
+WITH_MKL=ON WITH_AVX=ON WITH_TESTING=ON python setup.py install 
+# WITH_MKL=ON WITH_AVX=ON PY_VERSION=3.8 /bin/bash paddle/scripts/paddle_build.sh build_only ${MAX_JOBS}
+# find ./dist -name *whl | xargs pip install
+
+popd

ci/templates/musa.yaml

@@ -0,0 +1,80 @@
+kind: Pod
+apiVersion: v1
+spec:
+  # The shared volume named "artifacts".
+  # NOTE: `emptyDir: { }` declares an empty temporary volume, and
+  #       will be removed after all jobs are finished.
+  volumes:
+  - name: artifacts
+    emptyDir: { }
+  - name: inference-demo-data
+    hostPath:
+      path: /data/paddle_musa/inference_demo
+      type: DirectoryOrCreate
+  - name: shm-volume
+    emptyDir:
+      medium: Memory
+      sizeLimit: 2G
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: gputype
+            operator: In
+            values:
+            - mthreads-train
+  containers:
+  - name: main
+    tty: true
+    image: sh-harbor.mthreads.com/mt-ai/musa-paddle-dev:v0.1.0
+    imagePullPolicy: Always
+    env:
+    - name: PADDLE_MUSA_REPO_PATH
+      value: /home/paddle_musa
+    - name: WITH_MUSA
+      value: ON
+    - name: MUSA_VISIBLE_DEVICES
+      value: 1,2,3
+    - name: MTHREADS_VISIBLE_DEVICES
+      value: all
+    command:
+    - sleep
+    securityContext:
+       privileged: true
+    args:
+    - infinity
+    resources:
+      limits:
+        cpu: "16"
+        memory: "64Gi"
+    # Mount the volume named "artifacts" to path "/artifacts", readwrite
+    volumeMounts:
+    - mountPath: /artifacts
+      name: artifacts
+      readOnly: false
+    - mountPath: /dev/shm
+      name: shm-volume
+    - mountPath: /home/data/paddle_musa/.cache/build
+      name: inference-demo-data
+      readOnly: true
+
+  # Container named "release".
+  - name: release
+    image: sh-harbor.mthreads.com/mt-ai/ops/torch_musa_oss_release:latest
+    imagePullPolicy: IfNotPresent
+    tty: true
+    ttyEnabled: true
+    command:
+    - sleep
+    args:
+    - infinity
+    resources:
+      limits:
+        cpu: 300m
+        memory: 300Mi
+    # Mount the volume named "artifacts" to path "/artifacts", readwrite
+    volumeMounts:
+    - mountPath: /artifacts
+      name: artifacts
+      readOnly: false