From 4820c20c5ddb1edd7c594db1703f54c2f661996c Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 21:48:23 -0400 Subject: [PATCH 1/9] unittest_windows_cpu and unittest_windows_gpu [ghstack-poisoned] --- .circleci/config.yml | 150 +++++++++--------- .../unittest/windows/scripts/environment.yml | 7 + .circleci/unittest/windows/scripts/install.sh | 31 ++++ .../windows/scripts/install_conda.bat | 1 + .../unittest/windows/scripts/post_process.sh | 8 + .../unittest/windows/scripts/run_test.sh | 9 ++ .../unittest/windows/scripts/setup_env.sh | 39 +++++ .../windows/scripts/vc_env_helper.bat | 39 +++++ 8 files changed, 209 insertions(+), 75 deletions(-) create mode 100644 .circleci/unittest/windows/scripts/environment.yml create mode 100644 .circleci/unittest/windows/scripts/install.sh create mode 100644 .circleci/unittest/windows/scripts/install_conda.bat create mode 100644 .circleci/unittest/windows/scripts/post_process.sh create mode 100644 .circleci/unittest/windows/scripts/run_test.sh create mode 100644 .circleci/unittest/windows/scripts/setup_env.sh create mode 100644 .circleci/unittest/windows/scripts/vc_env_helper.bat diff --git a/.circleci/config.yml b/.circleci/config.yml index d45d0c9..290b4ac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -306,81 +306,81 @@ jobs: - build-csprng-develop - run-tests -# unittest_windows_cpu: -# <<: *binary_common -# executor: -# name: windows-cpu -# steps: -# - checkout -# - run: -# name: Generate cache key -# # This will refresh cache on Sundays, nightly build should generate new cache. -# command: echo "$(date +"%Y-%U")" > .circleci-weekly -# - restore_cache: -# -# keys: -# - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} -# -# - run: -# name: Setup -# command: .circleci/unittest/windows/scripts/setup_env.sh -# - save_cache: -# -# key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} -# -# paths: -# - conda -# - env -# - run: -# name: Install torchvision -# command: .circleci/unittest/windows/scripts/install.sh -# - run: -# name: Run tests -# command: .circleci/unittest/windows/scripts/run_test.sh -# - run: -# name: Post process -# command: .circleci/unittest/windows/scripts/post_process.sh -# - store_test_results: -# path: test-results -# -# unittest_windows_gpu: -# <<: *binary_common -# executor: -# name: windows-gpu -# environment: -# CUDA_VERSION: "10.1" -# steps: -# - checkout -# - run: -# name: Generate cache key -# # This will refresh cache on Sundays, nightly build should generate new cache. -# command: echo "$(date +"%Y-%U")" > .circleci-weekly -# - restore_cache: -# -# keys: -# - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} -# -# - run: -# name: Setup -# command: .circleci/unittest/windows/scripts/setup_env.sh -# - save_cache: -# -# key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} -# -# paths: -# - conda -# - env -# - run: -# name: Install torchvision -# command: .circleci/unittest/windows/scripts/install.sh -# - run: -# name: Run tests -# command: .circleci/unittest/windows/scripts/run_test.sh -# - run: -# name: Post process -# command: .circleci/unittest/windows/scripts/post_process.sh -# - store_test_results: -# path: test-results + unittest_windows_cpu: + <<: *binary_common + executor: + name: windows-cpu + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/windows/scripts/setup_env.sh + - save_cache: + + key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchcsprng + command: .circleci/unittest/windows/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/windows/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/windows/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_windows_gpu: + <<: *binary_common + executor: + name: windows-gpu + environment: + CUDA_VERSION: "10.1" + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/windows/scripts/setup_env.sh + - save_cache: + + key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchcsprng + command: .circleci/unittest/windows/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/windows/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/windows/scripts/post_process.sh + - store_test_results: + path: test-results workflows: build: diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml new file mode 100644 index 0000000..be3ad88 --- /dev/null +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -0,0 +1,7 @@ +channels: + - defaults +dependencies: + - numpy + - scipy + - pytest + diff --git a/.circleci/unittest/windows/scripts/install.sh b/.circleci/unittest/windows/scripts/install.sh new file mode 100644 index 0000000..deba8f6 --- /dev/null +++ b/.circleci/unittest/windows/scripts/install.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -e + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="cudatoolkit=${version}" +fi +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" + +printf "* Installing torchcsprng\n" +"$this_dir/vc_env_helper.bat" python setup.py develop \ No newline at end of file diff --git a/.circleci/unittest/windows/scripts/install_conda.bat b/.circleci/unittest/windows/scripts/install_conda.bat new file mode 100644 index 0000000..6612fba --- /dev/null +++ b/.circleci/unittest/windows/scripts/install_conda.bat @@ -0,0 +1 @@ +start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda% \ No newline at end of file diff --git a/.circleci/unittest/windows/scripts/post_process.sh b/.circleci/unittest/windows/scripts/post_process.sh new file mode 100644 index 0000000..2a1ac63 --- /dev/null +++ b/.circleci/unittest/windows/scripts/post_process.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +#codecov diff --git a/.circleci/unittest/windows/scripts/run_test.sh b/.circleci/unittest/windows/scripts/run_test.sh new file mode 100644 index 0000000..02c6327 --- /dev/null +++ b/.circleci/unittest/windows/scripts/run_test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/Scripts/conda.exe 'shell.bash' 'hook')" +conda activate ./env + +python -m torch.utils.collect_env +pytest --cov=torchcsprng --junitxml=test-results/junit.xml -v --durations 20 test \ No newline at end of file diff --git a/.circleci/unittest/windows/scripts/setup_env.sh b/.circleci/unittest/windows/scripts/setup_env.sh new file mode 100644 index 0000000..6a73927 --- /dev/null +++ b/.circleci/unittest/windows/scripts/setup_env.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchcsprng here, otherwise they also get cached. + +set -e + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + export tmp_conda="$(echo $conda_dir | tr '/' '\\')" + export miniconda_exe="$(echo $root_dir | tr '/' '\\')\\miniconda.exe" + curl --output miniconda.exe https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -O + "$this_dir/install_conda.bat" + unset tmp_conda + unset miniconda_exe +fi + +eval "$(${conda_dir}/Scripts/conda.exe 'shell.bash' 'hook')" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +conda env update --file "${this_dir}/environment.yml" --prune \ No newline at end of file diff --git a/.circleci/unittest/windows/scripts/vc_env_helper.bat b/.circleci/unittest/windows/scripts/vc_env_helper.bat new file mode 100644 index 0000000..9410135 --- /dev/null +++ b/.circleci/unittest/windows/scripts/vc_env_helper.bat @@ -0,0 +1,39 @@ +@echo on + +set VC_VERSION_LOWER=16 +set VC_VERSION_UPPER=17 + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do ( + if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" ( + set "VS15INSTALLDIR=%%i" + set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat" + goto vswhere + ) +) + +:vswhere +if "%VSDEVCMD_ARGS%" == "" ( + call "%VS15VCVARSALL%" x64 || exit /b 1 +) else ( + call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1 +) + +@echo on + +set DISTUTILS_USE_SDK=1 + +set args=%1 +shift +:start +if [%1] == [] goto done +set args=%args% %1 +shift +goto start + +:done +if "%args%" == "" ( + echo Usage: vc_env_helper.bat [command] [args] + echo e.g. vc_env_helper.bat cl /c test.cpp +) + +%args% || exit /b 1 From 8a84990042543dac137d3c846a311e4913c7d987 Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 21:49:53 -0400 Subject: [PATCH 2/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 68 ++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 290b4ac..cb77dd0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -772,40 +772,40 @@ workflows: cu_version: cu101 name: unittest_linux_gpu_py3.8 python_version: '3.8' -# - unittest_windows_cpu: -# cu_version: cpu -# name: unittest_windows_cpu_py3.6 -# python_version: '3.6' -# - unittest_windows_cpu: -# cu_version: cpu -# name: unittest_windows_cpu_py3.7 -# python_version: '3.7' -# - unittest_windows_cpu: -# cu_version: cpu -# name: unittest_windows_cpu_py3.8 -# python_version: '3.8' -# - unittest_windows_gpu: -# cu_version: cu101 -# filters: -# branches: -# only: -# - master -# - nightly -# name: unittest_windows_gpu_py3.6 -# python_version: '3.6' -# - unittest_windows_gpu: -# cu_version: cu101 -# filters: -# branches: -# only: -# - master -# - nightly -# name: unittest_windows_gpu_py3.7 -# python_version: '3.7' -# - unittest_windows_gpu: -# cu_version: cu101 -# name: unittest_windows_gpu_py3.8 -# python_version: '3.8' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.6 + python_version: '3.6' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.7 + python_version: '3.7' + - unittest_windows_cpu: + cu_version: cpu + name: unittest_windows_cpu_py3.8 + python_version: '3.8' + - unittest_windows_gpu: + cu_version: cu101 + filters: + branches: + only: + - master + - nightly + name: unittest_windows_gpu_py3.6 + python_version: '3.6' + - unittest_windows_gpu: + cu_version: cu101 + filters: + branches: + only: + - master + - nightly + name: unittest_windows_gpu_py3.7 + python_version: '3.7' + - unittest_windows_gpu: + cu_version: cu101 + name: unittest_windows_gpu_py3.8 + python_version: '3.8' nightly: jobs: # - circleci_consistency From ca992628a353621b83d1cf631f7c05cdb9e3d5a1 Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 21:56:56 -0400 Subject: [PATCH 3/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/unittest/windows/scripts/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index be3ad88..e72451e 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -4,4 +4,4 @@ dependencies: - numpy - scipy - pytest - + - ca-certificates From 67f538a9fae40a220da63009a25a98ab35f5a061 Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:04:31 -0400 Subject: [PATCH 4/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/unittest/windows/scripts/environment.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index e72451e..a8bb755 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -3,5 +3,3 @@ channels: dependencies: - numpy - scipy - - pytest - - ca-certificates From 17b37f8f23f5b3772d81abdf9787a6dbdc93953a Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:13:52 -0400 Subject: [PATCH 5/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 44 +++++++++---------- .../unittest/windows/scripts/environment.yml | 13 +++++- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cb77dd0..9313962 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -784,28 +784,28 @@ workflows: cu_version: cpu name: unittest_windows_cpu_py3.8 python_version: '3.8' - - unittest_windows_gpu: - cu_version: cu101 - filters: - branches: - only: - - master - - nightly - name: unittest_windows_gpu_py3.6 - python_version: '3.6' - - unittest_windows_gpu: - cu_version: cu101 - filters: - branches: - only: - - master - - nightly - name: unittest_windows_gpu_py3.7 - python_version: '3.7' - - unittest_windows_gpu: - cu_version: cu101 - name: unittest_windows_gpu_py3.8 - python_version: '3.8' +# - unittest_windows_gpu: +# cu_version: cu101 +# filters: +# branches: +# only: +# - master +# - nightly +# name: unittest_windows_gpu_py3.6 +# python_version: '3.6' +# - unittest_windows_gpu: +# cu_version: cu101 +# filters: +# branches: +# only: +# - master +# - nightly +# name: unittest_windows_gpu_py3.7 +# python_version: '3.7' +# - unittest_windows_gpu: +# cu_version: cu101 +# name: unittest_windows_gpu_py3.8 +# python_version: '3.8' nightly: jobs: # - circleci_consistency diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index a8bb755..ace13dc 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -2,4 +2,15 @@ channels: - defaults dependencies: - numpy - - scipy + - pytest + - pytest-cov + - codecov + - pip + - libpng + - jpeg + - ca-certificates + - pip: + - future + - pillow>=4.1.1 + - scipy==1.4.1 + - av From 342eeb9e80b8fd309fdaf7406d3925025c7cfdc8 Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:26:37 -0400 Subject: [PATCH 6/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 88 +++++++++++++++---- .../unittest/linux/scripts/environment.yml | 14 +++ .circleci/unittest/linux/scripts/install.sh | 29 ++++++ .../unittest/linux/scripts/post_process.sh | 8 ++ .circleci/unittest/linux/scripts/run_test.sh | 9 ++ .circleci/unittest/linux/scripts/setup_env.sh | 34 +++++++ .../unittest/windows/scripts/environment.yml | 2 - 7 files changed, 164 insertions(+), 20 deletions(-) create mode 100644 .circleci/unittest/linux/scripts/environment.yml create mode 100755 .circleci/unittest/linux/scripts/install.sh create mode 100755 .circleci/unittest/linux/scripts/post_process.sh create mode 100755 .circleci/unittest/linux/scripts/run_test.sh create mode 100755 .circleci/unittest/linux/scripts/setup_env.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 9313962..2668cb2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,6 +279,7 @@ jobs: for pkg in ~/workspace/*.whl; do aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/<< parameters.subfolder >>" --acl public-read done + unittest_linux_cpu: <<: *binary_common docker: @@ -286,25 +287,76 @@ jobs: resource_class: 2xlarge+ steps: - checkout - - install-conda - - install-pytorch - - build-csprng-develop - - run-tests + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: - unittest_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-1604-cuda-10.1:201909-23 - resource_class: gpu.small - environment: - image_name: "pytorch/manylinux-cuda101" - steps: - - checkout - - install-conda - - install-pytorch - - install-cuda - - build-csprng-develop - - run-tests + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchvision + command: .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post process + command: .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results + + unittest_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-1604-cuda-10.1:201909-23 + resource_class: gpu.small + environment: + image_name: "pytorch/manylinux-cuda101" + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: + + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + - run: + name: Setup + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: + + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + + paths: + - conda + - env + - run: + name: Install torchvision + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post Process + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results unittest_windows_cpu: <<: *binary_common diff --git a/.circleci/unittest/linux/scripts/environment.yml b/.circleci/unittest/linux/scripts/environment.yml new file mode 100644 index 0000000..7310ae6 --- /dev/null +++ b/.circleci/unittest/linux/scripts/environment.yml @@ -0,0 +1,14 @@ +channels: + - defaults +dependencies: + - numpy + - pytest + - pytest-cov + - codecov + - pip + - ca-certificates + - pip: + - future + - pillow>=4.1.1 + - scipy + - av diff --git a/.circleci/unittest/linux/scripts/install.sh b/.circleci/unittest/linux/scripts/install.sh new file mode 100755 index 0000000..6334cb9 --- /dev/null +++ b/.circleci/unittest/linux/scripts/install.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +unset PYTORCH_VERSION +# For unittest, nightly PyTorch is used as the following section, +# so no need to set PYTORCH_VERSION. +# In fact, keeping PYTORCH_VERSION forces us to hardcode PyTorch version in config. + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +if [ "${CU_VERSION:-}" == cpu ] ; then + cudatoolkit="cpuonly" +else + if [[ ${#CU_VERSION} -eq 4 ]]; then + CUDA_VERSION="${CU_VERSION:2:1}.${CU_VERSION:3:1}" + elif [[ ${#CU_VERSION} -eq 5 ]]; then + CUDA_VERSION="${CU_VERSION:2:2}.${CU_VERSION:4:1}" + fi + echo "Using CUDA $CUDA_VERSION as determined by CU_VERSION" + version="$(python -c "print('.'.join(\"${CUDA_VERSION}\".split('.')[:2]))")" + cudatoolkit="cudatoolkit=${version}" +fi +printf "Installing PyTorch with %s\n" "${cudatoolkit}" +conda install -y -c pytorch-nightly pytorch "${cudatoolkit}" + +printf "* Installing torchcsprng\n" +python setup.py develop \ No newline at end of file diff --git a/.circleci/unittest/linux/scripts/post_process.sh b/.circleci/unittest/linux/scripts/post_process.sh new file mode 100755 index 0000000..b05be6d --- /dev/null +++ b/.circleci/unittest/linux/scripts/post_process.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +codecov \ No newline at end of file diff --git a/.circleci/unittest/linux/scripts/run_test.sh b/.circleci/unittest/linux/scripts/run_test.sh new file mode 100755 index 0000000..61f6e3e --- /dev/null +++ b/.circleci/unittest/linux/scripts/run_test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e + +eval "$(./conda/bin/conda shell.bash hook)" +conda activate ./env + +python -m torch.utils.collect_env +pytest --cov=torchcsprng --junitxml=test-results/junit.xml -v --durations 20 test \ No newline at end of file diff --git a/.circleci/unittest/linux/scripts/setup_env.sh b/.circleci/unittest/linux/scripts/setup_env.sh new file mode 100755 index 0000000..303042f --- /dev/null +++ b/.circleci/unittest/linux/scripts/setup_env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +# This script is for setting up environment in which unit test is ran. +# To speed up the CI time, the resulting environment is cached. +# +# Do not install PyTorch and torchcsprng here, otherwise they also get cached. + +set -e + +this_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +root_dir="$(git rev-parse --show-toplevel)" +conda_dir="${root_dir}/conda" +env_dir="${root_dir}/env" + +cd "${root_dir}" + +# 1. Install conda at ./conda +if [ ! -d "${conda_dir}" ]; then + printf "* Installing conda\n" + wget -O miniconda.sh http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh + bash ./miniconda.sh -b -f -p "${conda_dir}" +fi +eval "$(${conda_dir}/bin/conda shell.bash hook)" + +# 2. Create test environment at ./env +if [ ! -d "${env_dir}" ]; then + printf "* Creating a test environment\n" + conda create --prefix "${env_dir}" -y python="$PYTHON_VERSION" +fi +conda activate "${env_dir}" + +# 3. Install Conda dependencies +printf "* Installing dependencies (except PyTorch)\n" +conda env update --file "${this_dir}/environment.yml" --prune diff --git a/.circleci/unittest/windows/scripts/environment.yml b/.circleci/unittest/windows/scripts/environment.yml index ace13dc..52593ad 100644 --- a/.circleci/unittest/windows/scripts/environment.yml +++ b/.circleci/unittest/windows/scripts/environment.yml @@ -6,8 +6,6 @@ dependencies: - pytest-cov - codecov - pip - - libpng - - jpeg - ca-certificates - pip: - future From 798ee418ed57e90c6b3893b82f93cf140107cfa8 Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:27:55 -0400 Subject: [PATCH 7/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2668cb2..e8cbdb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,48 +42,6 @@ commands: fi echo "export UPLOAD_CHANNEL=${our_upload_channel}" >> ${BASH_ENV} - install-conda: - steps: - - run: - name: Installing Conda - command: | - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh - bash Miniconda3-latest-Linux-x86_64.sh -b -p $HOME/conda - export PATH=$HOME/conda/bin:$PATH - echo 'export PATH=$HOME/conda/bin:$PATH' >> $BASH_ENV - conda install -y python=3 - - install-pytorch: - steps: - - run: - name: Installing PyTorch - command: | - conda install pytorch -yc pytorch-nightly - conda install -y numpy scipy - - install-cuda: - steps: - - run: - name: Installing CUDA - command: | - wget http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda-repo-ubuntu1604-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb - sudo dpkg -i cuda-repo-ubuntu1604-10-2-local-10.2.89-440.33.01_1.0-1_amd64.deb - sudo apt-key add /var/cuda-repo-10-2-local-10.2.89-440.33.01/7fa2af80.pub - sudo apt-get update - sudo apt-get -y install cuda - - build-csprng-develop: - steps: - - run: - name: Building CSPRNG - command: python setup.py develop - - run-tests: - steps: - - run: - name: Running tests - command: python setup.py test - binary_common: &binary_common parameters: # Edit these defaults to do a release` From ca286587a6bd5ebcfa80bd187ffe36d3eb01113a Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:29:54 -0400 Subject: [PATCH 8/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 70 ++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8cbdb7..793e559 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,45 +276,45 @@ jobs: - store_test_results: path: test-results - unittest_linux_gpu: - <<: *binary_common - machine: - image: ubuntu-1604-cuda-10.1:201909-23 - resource_class: gpu.small - environment: - image_name: "pytorch/manylinux-cuda101" - steps: - - checkout - - run: - name: Generate cache key - # This will refresh cache on Sundays, nightly build should generate new cache. - command: echo "$(date +"%Y-%U")" > .circleci-weekly - - restore_cache: + unittest_linux_gpu: + <<: *binary_common + machine: + image: ubuntu-1604-cuda-10.1:201909-23 + resource_class: gpu.small + environment: + image_name: "pytorch/manylinux-cuda101" + steps: + - checkout + - run: + name: Generate cache key + # This will refresh cache on Sundays, nightly build should generate new cache. + command: echo "$(date +"%Y-%U")" > .circleci-weekly + - restore_cache: - keys: - - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + keys: + - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - - run: - name: Setup - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh - - save_cache: + - run: + name: Setup + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh + - save_cache: - key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} + key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - paths: - - conda - - env - - run: - name: Install torchvision - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh - - run: - name: Run tests - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh - - run: - name: Post Process - command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh - - store_test_results: - path: test-results + paths: + - conda + - env + - run: + name: Install torchvision + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh + - run: + name: Run tests + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/run_test.sh + - run: + name: Post Process + command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/post_process.sh + - store_test_results: + path: test-results unittest_windows_cpu: <<: *binary_common From 95ca73f13b1940b54d47a69329c9a4a14858343e Mon Sep 17 00:00:00 2001 From: Pavel Belevich Date: Thu, 13 Aug 2020 22:40:35 -0400 Subject: [PATCH 9/9] Update on "unittest_windows_cpu and unittest_windows_gpu" [ghstack-poisoned] --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 793e559..f01ff6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -265,7 +265,7 @@ jobs: - conda - env - run: - name: Install torchvision + name: Install torchcsprng command: .circleci/unittest/linux/scripts/install.sh - run: name: Run tests @@ -305,7 +305,7 @@ jobs: - conda - env - run: - name: Install torchvision + name: Install torchcsprng command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/install.sh - run: name: Run tests