bobby/open_dbm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[New Feature] CICD Implementation for build and testing OpenDBM in all platforms

Browse Source
This commit is contained in:
Rudy Haryanto
2022-10-04 02:19:07 +07:00
parent a574bc6870
commit 67618c66bc
18 changed files with 857 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/actions/win-install-sox/action.yml vendored Normal file
View File

@@ -0,0 +1,16 @@
#"C:/Program Files/7-Zip/7z.exe"
name: "Install SoX and add to PATH"
description: "Install SoX and add to PATH"
runs:
using: "composite"
steps:
- shell: bash
run: |
curl -L --output sox-14.4.2-win32.zip --url https://sourceforge.net/projects/sox/files/sox/14.4.2/sox-14.4.2-win32.zip/download
7z x -o`pwd`/bin/ -tzip -aoa sox-14.4.2-win32.zip
echo `pwd`
rm sox-*zip

285
.github/workflows/open_dbm-build-checking.yml vendored Normal file
View File

@@ -0,0 +1,285 @@
# This workflow will check OpenDMB code quality, run unit tests and update the build status on README.md
# For more information see: https://www.aicure.com/open_dbm#cicd
name: OpenDBM Build Checking
on:
push:
branches: [ master ]
paths:
- 'dbm_lib/**'
- 'opendbm/**'
- '.github/workflows/open_dbm-build-checking.yml'
- '.github/workflows/open_dbm-code-checking.yml'
- 'tests/**'
- 'pkg/**'
- 'resources/**'
- 'requirements.txt'
workflow_dispatch:
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
jobs:
unit_tests_linux:
runs-on: ubuntu-latest
steps:
- name: 🔍 GH_TOKEN
if: env.GH_TOKEN == ''
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "GH_TOKEN=${GITHUB_TOKEN}" >> $GITHUB_ENV
- name: Checkout OpenDBM code
uses: actions/checkout@v3
with:
token: ${{ env.GH_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- uses: actions/cache@v3
id: cache
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Pull OpenFace Image
run: |
docker pull ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1.1.0
- name: Install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
pip install -e .[test]
# ls not_exist_file #####commented because of unknown command and it make the build error
- name : Install libsndfile and sox(only for linux machine)
run: |
sudo apt-get install libsndfile1
sudo apt-get install sox
- name: Run Pytest
id: odbm_linux_test
run: |
pytest
- name: Update shield LINUX BUILD status only for branch push master
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && always() }}
run: |
echo "${{ steps.odbm_linux_test.outcome }}"
set -x
test_status='${{ steps.odbm_linux_test.outcome }}'
if [ "$test_status" = "success" ] ; then
COLOR=green
STATUS=passed
else
COLOR=red
STATUS=failed
fi
curl "https://img.shields.io/badge/linux:build-$STATUS-$COLOR?style=flat&logo=linux" > images/badges/linux_status.svg
git config --global user.name 'AiCure ODBM Pipeline'
git config --global user.email 'system@aicure.com'
if [[ `git status --porcelain` ]]; then
git pull origin master
git add images/badges/linux_status.svg
git commit -am "Modify Linux Build status in badge"
git push
else
echo "no changes on linux build badge"
fi
unit_tests_windows:
runs-on: windows-latest
steps:
- name: 🔍 GH_TOKEN
if: env.GH_TOKEN == ''
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "GH_TOKEN=${GITHUB_TOKEN}" >> $GITHUB_ENV
- name: Checkout OpenDBM code
uses: actions/checkout@v3
with:
token: ${{ env.GH_TOKEN }}
# - name: install WSL distribution
# uses: Vampire/setup-wsl@v1
# with:
# distribution: Ubuntu-18.04
# - name: update wsl
# run: |
#
# wsl --install
# wsl --list
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v2
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
# - name: Login to Docker Hub
# uses: docker/login-action@v2
# with:
# username: ${{ secrets.DOCKERHUB_USERNAME }}
# password: ${{ secrets.DOCKERHUB_TOKEN }}
# - name: Pull OpenFace Image
# run: |
# docker pull ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest
# docker images
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- uses: actions/cache@v3
id: cache
with:
path: ${{ env.pythonLocation }}
key: windows-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
- name: Switch git-bash shell to MSYS2 shell by adding MSYS2 path to PATH front
run: echo "$MSYS2_SHELL_PATH" >> $GITHUB_PATH
- name: Install SoX and add to Path
uses: ./.github/actions/win-install-sox
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1.1.0
- name: add sox to PATH
uses: myci-actions/export-env-var-powershell@1
with:
name: PATH
value: $env:PATH;D:/a/${{ github.event.repository.name }}/${{ github.event.repository.name }}/bin/sox-14.4.2/
- name: Install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
pip install -e .[test]
pip install https://github.com/sachadee/Dlib/blob/main/dlib-19.22.99-cp37-cp37m-win_amd64.whl?raw=true
- name: Run Pytest
id: odbm_windows_test
run: |
pytest -m "speech or acoustic"
- name: Update shield WINDOWS BUILD status only for branch push master
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && always() }}
run: |
Write-Host "${{ steps.odbm_windows_test.outcome }}";
$test_status='${{ steps.odbm_windows_test.outcome }}'
if($test_status -eq "success"){
$COLOR="green"
$STATUS="passed"
}else {
$COLOR="red"
$STATUS="failed"
}
$Parameters = @{
style = 'flat'
logo = 'windows'
}
Write-Host "$COLOR $STATUS"
Invoke-WebRequest -Uri "https://img.shields.io/badge/windows:build-$STATUS-$COLOR" -Body $Parameters -Method Get -PassThru -OutFile images/badges/windows_status.svg
git config --global user.name 'AiCure ODBM Pipeline'
git config --global user.email 'system@aicure.com'
if (git status --porcelain) {
git pull origin master
git add images/badges/windows_status.svg
git commit -am "Modify Windows Build status in badge"
git push
}else {
Write-Host "no changes on Windows build badge"
}
unit_tests_macos:
runs-on: macos-latest
steps:
- name: 🔍 GH_TOKEN
if: env.GH_TOKEN == ''
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "GH_TOKEN=${GITHUB_TOKEN}" >> $GITHUB_ENV
- name: Checkout OpenDBM code
uses: actions/checkout@v3
with:
token: ${{ env.GH_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- uses: actions/cache@v3
id: cache
with:
path: ${{ env.pythonLocation }}
key: macos-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
- name: setup docker
uses: douglascamata/setup-docker-macos-action@v1-alpha
# docker-practice/actions-setup-docker@master
- name: check docker
run: |
set -x
docker version
docker run --rm hello-world
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Pull OpenFace Image
run: |
docker pull ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest
- name: Setup FFmpeg
uses: FedericoCarboni/setup-ffmpeg@v1.1.0
- name: Install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
pip install -e .[test]
- name : Install sox
run: brew install sox
- name: Run Pytest
id: odbm_macos_test
run: |
pytest
- name: Update shield MACOS BUILD status only for branch push master
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && always() }}
run: |
echo "${{ steps.odbm_macos_test.outcome }}"
set -x
test_status='${{ steps.odbm_macos_test.outcome }}'
if [ "$test_status" = "success" ] ; then
COLOR=green
STATUS=passed
else
COLOR=red
STATUS=failed
fi
curl "https://img.shields.io/badge/macos:build-$STATUS-$COLOR?style=flat&logo=apple" > images/badges/macos_status.svg
git config --global user.name 'AiCure ODBM Pipeline'
git config --global user.email 'system@aicure.com'
if [[ `git status --porcelain` ]]; then
git pull origin master
git add images/badges/macos_status.svg
git commit -am "Modify MacOs Build status in badge"
git push
else
echo "no changes on MacOs build badge"
fi
update_status_badge_master:
runs-on: ubuntu-latest
needs: [unit_tests_linux, unit_tests_windows, unit_tests_macos]
if: |
always()
steps:
- name: 🔍 GH_TOKEN
if: env.GH_TOKEN == ''
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "GH_TOKEN=${GITHUB_TOKEN}" >> $GITHUB_ENV
- name: Checkout OpenDBM code
uses: actions/checkout@v3
with:
token: ${{ env.GH_TOKEN }}
- name: Purge README Images
run: make purge

175
.github/workflows/open_dbm-code-checking.yml vendored Normal file
View File

@@ -0,0 +1,175 @@
# This workflow will check OpenDMB code quality and run unit tests
# For more information see: https://www.aicure.com/open_dbm#cicd
name: OpenDBM Code Checking
on:
push:
branches: [ master ]
paths:
- 'dbm_lib/**'
- 'opendbm/**'
- '.github/workflows/open_dbm-code-checking.yml'
- '.github/workflows/open_dbm-build-checking.yml'
- 'tests/**'
- 'pkg/**'
- 'resources/**'
- 'requirements.txt'
pull_request:
branches: [ master ]
paths:
- 'dbm_lib/**'
- 'opendbm/**'
- '.github/workflows/open_dbm-code-checking.yml'
- '.github/workflows/open_dbm-build-checking.yml'
- 'tests/**'
- 'pkg/**'
- 'resources/**'
- 'requirements.txt'
workflow_dispatch:
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
jobs:
linting:
runs-on: ubuntu-latest
steps:
- name: Checkout OpenDBM code
uses: actions/checkout@v3
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@v23.1
- name: Report list of changed files
run: |
echo Changed files: ${{ steps.changed-files.outputs.all_changed_files }}
- name: Get PYTHON changed files
id: python-changed-files
uses: tj-actions/changed-files@v23.1
with:
files: |
*.py*
**/*.py*
- name: Report list of PYTHON changed files
run: |
echo Changed PYTHON files: ${{ steps.python-changed-files.outputs.all_changed_files }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Run flake8
run: |
pip install flake8
for file in ${{ steps.python-changed-files.outputs.all_changed_files }}; do
flake8 $file --count --show-source --statistics
done
vulnerability_scanning:
runs-on: ubuntu-latest
steps:
- name: Checkout OpenDBM code
uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Run safety
run: |
pip install safety
safety check
unit_tests:
runs-on: ubuntu-latest
steps:
- name: 🔍 GH_TOKEN
if: env.GH_TOKEN == ''
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: echo "GH_TOKEN=${GITHUB_TOKEN}" >> $GITHUB_ENV
- name: Checkout OpenDBM code
uses: actions/checkout@v3
with:
token: ${{ env.GH_TOKEN }}
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- uses: actions/cache@v3
id: cache
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('requirements.txt') }}
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Pull OpenFace Image
run: |
docker pull ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest
- name: Install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
pip install -e .[test]
- name : Install libsndfile and sox(only for linux machine)
run: |
sudo apt-get install libsndfile1
sudo apt-get install sox
sudo apt-get install ffmpeg
- name: Run Pytest and Coverage
id: odbm_test
run: |
ls -lart
pip install pytest coverage
coverage run -m pytest
coverage report -mi --fail-under=30 --skip-covered
- name: Update shield TEST status only for branch push master
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && always() }}
run: |
echo "${{ steps.odbm_test.outcome }}"
set -x
test_status='${{ steps.odbm_test.outcome }}'
if [ "$test_status" = "success" ] ; then
COLOR=green
STATUS=passed
else
COLOR=red
STATUS=failed
fi
curl "https://img.shields.io/badge/test:status-$STATUS-$COLOR?style=flat" > images/badges/test_status.svg
git config --global user.name 'AiCure ODBM Pipeline'
git config --global user.email 'system@aicure.com'
if [[ `git status --porcelain` ]]; then
git pull origin master
git add images/badges/test_status.svg
git commit -am "Modify test status in test badge"
git push
else
echo "no changes on test badge"
fi
- name: Update shield COVERAGE only for branch push master
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
run: |
set -x
total=`coverage report -i | grep TOTAL | awk '{print $4+0}'`
if (( $(echo "$total < 30" | bc -l) )) ; then
COLOR=red
elif (( $(echo "$total >= 60" | bc -l) )); then
COLOR=green
else
COLOR=orange
fi
curl "https://img.shields.io/badge/code:coverage-$total%25-$COLOR?style=flat" > images/badges/code_coverage.svg
git config --global user.name 'AiCure ODBM Pipeline'
git config --global user.email 'system@aicure.com'
if [[ `git status --porcelain` ]]; then
git pull origin master
git add images/badges/code_coverage.svg
git commit -am "Modify coverage percentage in coverage badge"
git push
else
echo "no changes on coverage badge"
fi

51
.github/workflows/open_dbm-docs-deploy.yml vendored Normal file
View File

@@ -0,0 +1,51 @@
# This workflow will deploy opendbm documentation web
# For more information see: https://www.aicure.com/open_dbm
name: OpenDBM Web Documentation Deployment
on:
workflow_dispatch:
jobs:
build:
runs-on: ubuntu-latest
defaults:
run:
working-directory: docs
steps:
- name: Checkout OpenDBM code
uses: actions/checkout@v3
- uses: actions/setup-node@v3
with:
node-version: 16.x
# cache: yarn
# cache-dependency-path: './docs/package-lock.json'
- name: Install dependencies
run: |
pip install pydoc-markdown
pydoc-markdown -I ../opendbm/api_lib/facial_activity -m api --render-toc > website/api/facial-activity-api.md
pydoc-markdown -I ../opendbm/api_lib/movement -m api --render-toc > website/api/movement-api.md
pydoc-markdown -I ../opendbm/api_lib/verbal_acoustics -m api --render-toc > website/api/verbal-acoustics-api.md
pydoc-markdown -I ../opendbm/api_lib/speech -m api --render-toc > website/api/speech-api.md
yarn install --frozen-lockfile
- name: Build website
run: |
ls -alrt
cd website
yarn build
# Popular action to deploy to GitHub Pages:
# Docs: https://github.com/peaceiris/actions-gh-pages#%EF%B8%8F-docusaurus
- name: Deploy to GitHub Pages
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
# Build output to publish to the `gh-pages` branch:
publish_dir: docs/website/build
# The following lines assign commit authorship to the official
# GH-Actions bot for deploys to `gh-pages` branch:
# https://github.com/actions/checkout/issues/13#issuecomment-724415212
# The GH actions bot is used by default if you didn't specify the two fields.
# You can swap them out with your own user credentials.
user_name: github-actions[bot]
user_email: 41898282+github-actions[bot]@users.noreply.github.com

52
.github/workflows/open_dbm-publish-release.yml vendored Normal file
View File

@@ -0,0 +1,52 @@
# This workflow will publish OpenDBM to PyPI based on created release tag
# For more information see: https://www.aicure.com/open_dbm#cicd
name: OpenDBM Publish Release
on:
release:
types: [published]
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}
jobs:
release-job:
runs-on: ubuntu-latest
name: Build and publish OpenDBM distributions 📦 to PyPI
if: github.event_name == 'release' && github.event.action == 'published'
steps:
- name: Checkout OpenDBM code
uses: actions/checkout@v3
# with:
# fetch-depth: 2
# ref: ${{ github.ref }}
# token: ${{ env.GH_TOKEN }}
# - run: git checkout HEAD^
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.7
- uses: actions/cache@v3
id: cache
with:
path: ${{ env.pythonLocation }}
key: ${{ env.pythonLocation }}-${{ hashFiles('setup.cfg') }}-${{ hashFiles('requirements.txt') }}
- name: Install dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: |
python -m pip install --upgrade pip
pip install wheel build twine
pip install -e .
- name: Builds and uploads to PyPI
## can use python3 instead python if didnt work
run: |
echo "GITHUB REFNAME"
echo "${{ github.ref_name }}"
tag='${{ github.ref_name }}'
sed -Ei "s/^(version[[:blank:]]*=[[:blank:]]*).*/\1$tag/" setup.cfg
python -m build
twine upload dist/*
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}

31
.github/workflows/open_dbm_build_openface.yml vendored Normal file
View File

@@ -0,0 +1,31 @@
name: CI-Dockerhub-upload-template
on:
workflow_dispatch:
jobs:
docker:
runs-on: ubuntu-latest
steps:
-
name: Checkout
uses: actions/checkout@v3
-
name: Set up QEMU
uses: docker/setup-qemu-action@v2
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
-
name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
# -
# name: Docker build & push
# run: |
# docker pull dbm-test2
# docker image tag dbm-test2 ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest
# docker push ${{ secrets.DOCKERHUB_USERNAME }}/dbm-openface:latest