diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d843e4cc7d72..d136f0cc7e48 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,16 +23,16 @@ jobs: DJANGO_SU_EMAIL: 'admin@localhost.company' DJANGO_SU_PASSWORD: '12qwaszx' run: | - docker-compose -f docker-compose.yml -f docker-compose.ci.yml build - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' - docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info' + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}' + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info' docker-compose up -d docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" - name: Code instrumentation run: | npm ci npm run coverage - docker-compose up -d --build + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build - name: End-to-end testing run: | cd ./tests diff --git a/.github/workflows/publish_docker_images.yml b/.github/workflows/publish_docker_images.yml new file mode 100644 index 000000000000..44aad7e9137b --- /dev/null +++ b/.github/workflows/publish_docker_images.yml @@ -0,0 +1,54 @@ +name: Publish Docker images +on: + release: + types: [published] + +jobs: + build_and_push_to_registry: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + node-version: 12 + + - name: Build images + run: | + CLAM_AV=yes INSTALL_SOURCES=yes docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build + + - name: Run unit tests + env: + HOST_COVERAGE_DATA_DIR: ${{ github.workspace }} + CONTAINER_COVERAGE_DATA_DIR: '/coverage_data' + DJANGO_SU_NAME: 'admin' + DJANGO_SU_EMAIL: 'admin@localhost.company' + DJANGO_SU_PASSWORD: '12qwaszx' + run: | + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli' + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test' + docker-compose up -d + docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" + + - name: Run end-to-end tests + run: | + cd ./tests + npm ci + npx cypress run --headless --browser chrome + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push to Docker Hub + env: + DOCKERHUB_WORKSPACE: 'openvino' + SERVER_IMAGE_REPO: 'cvat_server' + UI_IMAGE_REPO: 'cvat_ui' + run: | + docker tag "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" + + docker tag "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}" + docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml index 16b7e9aba8de..ecce8547728e 100644 --- a/.github/workflows/schedule.yml +++ b/.github/workflows/schedule.yml @@ -18,7 +18,7 @@ jobs: DJANGO_SU_PASSWORD: "12qwaszx" API_ABOUT_PAGE: "localhost:8080/api/v1/server/about" run: | - docker-compose -f docker-compose.yml -f ./tests/docker-compose.email.yml up -d --build + docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f ./tests/docker-compose.email.yml up -d --build /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done' docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell" - name: End-to-end testing diff --git a/.remarkrc.js b/.remarkrc.js index e57d403a110f..5c301e63c101 100644 --- a/.remarkrc.js +++ b/.remarkrc.js @@ -3,7 +3,7 @@ exports.settings = { bullet: '*', paddedTable: false }; exports.plugins = [ 'remark-preset-lint-recommended', 'remark-preset-lint-consistent', - ['remark-preset-lint-markdown-style-guide', 'mixed'], + ['remark-lint-list-item-indent', 'space'], ['remark-lint-no-dead-urls', { skipOffline: true }], ['remark-lint-maximum-line-length', 120], ['remark-lint-maximum-heading-length', 120], diff --git a/CHANGELOG.md b/CHANGELOG.md index c764acae68f8..c7d73573ba26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Intelligent scissors with OpenCV javascript () - [Inside Outside Guidence](https://github.com/shiyinzhang/Inside-Outside-Guidance) serverless function for interative segmentation +- Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and + [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub () ### Changed diff --git a/Dockerfile b/Dockerfile index 741bb9df431d..9c38ec5d35fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,10 +34,12 @@ RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar. make -j5 && make install PREFIX=${PREFIX} && make clean WORKDIR /tmp/ffmpeg -RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ - tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \ +RUN curl -sL https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 --output - | \ + tar -jx --strip-components=1 && \ ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-shared --disable-static --prefix="${PREFIX}" && \ - make -j5 && make install && make distclean + # make clean keeps the configuration files that let to know how the original sources were used to create the binary + make -j5 && make install && make clean && \ + tar -zcf "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" . && mv "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" . # Install requirements RUN python3 -m venv /opt/venv @@ -93,8 +95,17 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* && \ echo 'application/wasm wasm' >> /etc/mime.types -ARG CLAM_AV -ENV CLAM_AV=${CLAM_AV} +# Add a non-root user +ENV USER=${USER} +ENV HOME /home/${USER} +RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ + if [ -z ${socks_proxy} ]; then \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ + else \ + echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ + fi + +ARG CLAM_AV="no" RUN if [ "$CLAM_AV" = "yes" ]; then \ apt-get update && \ apt-get --no-install-recommends install -yq \ @@ -106,16 +117,6 @@ RUN if [ "$CLAM_AV" = "yes" ]; then \ rm -rf /var/lib/apt/lists/*; \ fi -# Add a non-root user -ENV USER=${USER} -ENV HOME /home/${USER} -RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ - if [ -z ${socks_proxy} ]; then \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ - else \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ - fi - ARG INSTALL_SOURCES='no' WORKDIR ${HOME}/sources RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \ @@ -131,7 +132,7 @@ RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \ done && \ rm -rf /var/lib/apt/lists/*; \ fi -COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.bz2 ${HOME}/sources/ +COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.gz ${HOME}/sources/ # Copy python virtual enviroment and FFmpeg binaries from build-image COPY --from=build-image /opt/venv /opt/venv diff --git a/Dockerfile.ci b/Dockerfile.ci index bc642137e17b..4becdae086ed 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM cvat/server +FROM openvino/cvat_server ENV DJANGO_CONFIGURATION=testing USER root diff --git a/LICENSE b/LICENSE index e6959cbc514a..b4bd68239995 100644 --- a/LICENSE +++ b/LICENSE @@ -20,3 +20,12 @@ OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.   +This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project. +The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile). + +FFmpeg is an open source framework licensed under LGPL and GPL. +See https://www.ffmpeg.org/legal.html. You are solely responsible +for determining if your use of FFmpeg requires any +additional licenses. Intel is not responsible for obtaining any +such licenses, nor liable for any licensing fees due in +connection with your use of FFmpeg. \ No newline at end of file diff --git a/README.md b/README.md index 7137a54bc2ab..ed2ca0feb9cd 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,13 @@ Limitations: - No more than 10 tasks per user - Uploaded data is limited to 500Mb +## Prebuilt Docker images + +Prebuilt docker images for CVAT releases are available on Docker Hub: + +- [cvat_server](https://hub.docker.com/r/openvino/cvat_server) +- [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) + ## REST API Automatically generated Swagger documentation for Django REST API is available @@ -112,6 +119,16 @@ name. Example - `ALLOWED_HOSTS: 'localhost, 127.0.0.1'`. Code released under the [MIT License](https://opensource.org/licenses/MIT). +This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project. +The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile). + +FFmpeg is an open source framework licensed under LGPL and GPL. +See [https://www.ffmpeg.org/legal.html](https://www.ffmpeg.org/legal.html). You are solely responsible +for determining if your use of FFmpeg requires any +additional licenses. Intel is not responsible for obtaining any +such licenses, nor liable for any licensing fees due in +connection with your use of FFmpeg. + ## Questions CVAT usage related questions or unclear concepts can be posted in our diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md index da7c9c199bd0..fdd49faa22ae 100644 --- a/cvat/apps/documentation/installation.md +++ b/cvat/apps/documentation/installation.md @@ -81,18 +81,18 @@ server. Proxy is an advanced topic and it is not covered by the guide. cd cvat ``` -- Build docker images by default. It will take some time to download public - docker image ubuntu:16.04 and install all necessary ubuntu packages to run - CVAT server. +- Run docker containers. It will take some time to download the latest CVAT + release and other required images like postgres, redis, etc. from DockerHub and create containers. - ```bash - docker-compose build + ```sh + docker-compose up -d ``` -- Run docker containers. It will take some time to download public docker - images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers. +- Alternative: if you want to build the images locally with unreleased changes + run the following command. It will take some time to build CVAT images. - ```sh + ```bash + docker-compose -f docker-compose.yml -f docker-compose.dev.yml build docker-compose up -d ``` @@ -149,23 +149,23 @@ server. Proxy is an advanced topic and it is not covered by the guide. - Clone _CVAT_ source code from the [GitHub repository](https://github.com/opencv/cvat). - ```bash + ```sh git clone https://github.com/opencv/cvat cd cvat ``` -- Build docker images by default. It will take some time to download public - docker image ubuntu:16.04 and install all necessary ubuntu packages to run - CVAT server. +- Run docker containers. It will take some time to download the latest CVAT + release and other required images like postgres, redis, etc. from DockerHub and create containers. - ```bash - docker-compose build + ```sh + docker-compose up -d ``` -- Run docker containers. It will take some time to download public docker - images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers. +- Alternative: if you want to build the images locally with unreleased changes + run the following command. It will take some time to build CVAT images. ```sh + docker-compose -f docker-compose.yml -f docker-compose.dev.yml build docker-compose up -d ``` @@ -221,18 +221,18 @@ server. Proxy is an advanced topic and it is not covered by the guide. cd cvat ``` -- Build docker images by default. It will take some time to download public - docker image ubuntu:16.04 and install all necessary ubuntu packages to run - CVAT server. +- Run docker containers. It will take some time to download the latest CVAT + release and other required images like postgres, redis, etc. from DockerHub and create containers. - ```bash - docker-compose build + ```sh + docker-compose up -d ``` -- Run docker containers. It will take some time to download public docker - images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers. +- Alternative: if you want to build the images locally with unreleased changes + run the following command. It will take some time to build CVAT images. ```sh + docker-compose -f docker-compose.yml -f docker-compose.dev.yml build docker-compose up -d ``` diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index b46c960bea30..2b60e72b0c57 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,3 +1,8 @@ +# +# Copyright (C) 2018-2020 Intel Corporation +# +# SPDX-License-Identifier: MIT +# version: '3.3' services: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 000000000000..8a7510cafc09 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,31 @@ +# +# Copyright (C) 2021 Intel Corporation +# +# SPDX-License-Identifier: MIT +# +version: '3.3' + +services: + cvat: + build: + context: . + args: + http_proxy: + https_proxy: + no_proxy: nuclio,${no_proxy} + socks_proxy: + USER: 'django' + DJANGO_CONFIGURATION: 'production' + TZ: 'Etc/UTC' + CLAM_AV: + INSTALL_SOURCES: + + cvat_ui: + build: + context: . + args: + http_proxy: + https_proxy: + no_proxy: + socks_proxy: + dockerfile: Dockerfile.ui diff --git a/docker-compose.yml b/docker-compose.yml index 677177c39325..daa74ee90ae7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # -# Copyright (C) 2018-2020 Intel Corporation +# Copyright (C) 2018-2021 Intel Corporation # # SPDX-License-Identifier: MIT # @@ -32,22 +32,11 @@ services: cvat: container_name: cvat - image: cvat/server + image: openvino/cvat_server restart: always depends_on: - cvat_redis - cvat_db - build: - context: . - args: - http_proxy: - https_proxy: - no_proxy: nuclio,${no_proxy} - socks_proxy: - USER: 'django' - DJANGO_CONFIGURATION: 'production' - TZ: 'Etc/UTC' - CLAM_AV: 'no' environment: DJANGO_MODWSGI_EXTRA_ARGS: '' ALLOWED_HOSTS: '*' @@ -61,17 +50,8 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui + image: openvino/cvat_ui restart: always - build: - context: . - args: - http_proxy: - https_proxy: - no_proxy: - socks_proxy: - dockerfile: Dockerfile.ui - networks: default: aliases: