Commit a4d66d18 authored by Yakubov, Sergey's avatar Yakubov, Sergey
Browse files

Merge branch '5-get-file-using-outbound-mechanisms-only' into 'main' 

Resolve "Get file using outbound connections only"

See merge request !9
parents 6b594acf 7a644086

.coveragerc

deleted100644 → 0
+0 −6
Original line number Diff line number Diff line 
[report]

omit =

    tests/*

    remote_data_broker/brokers/filesys/*

    remote_data_broker/auth/keycloak.py

    remote_data_broker/main.py
 
 No newline at end of file

.flake8

deleted100644 → 0
+0 −2
Original line number Diff line number Diff line 
[flake8]

max-line-length = 120

.gitignore

+1 −8
Original line number Diff line number Diff line 
.idea

__pycache__

*~

*.log

*.bkp

.dockerignore

.coverage

htmlcov

coverage.xml

dist
 
 No newline at end of file
 
build
 
 No newline at end of file

.gitlab-ci.yml

+34 −94
Original line number Diff line number Diff line 
stages:

  - test

  - build

  - unit-tests

  - build-prod

  - integration-tests

  - update-repo



variables:

  GIT_STRATEGY: clone

  CONTAINER_RDM_URL: "${CI_REGISTRY_IMAGE}/remote-data-broker-dev"

  CONTAINER_RDM_URL_PROD: "${CI_REGISTRY_IMAGE}/remote-data-broker"

  coverage_report_view: "true"

  CONTAINER_RDM_URL: "${CI_REGISTRY_IMAGE}/remote-data-broker"

  TAG: 0.1.0



# This import is for the func_rse_docker_* functions

before_script:
@@ -25,118 +21,62 @@ after_script: 
  - func_rse_docker_cleanup

  - sudo chown -R gitlab-runner .



lint:

  stage: test

image-build:

  stage: build

  script:

    - >

      docker build

      -f dockerfiles/Dockerfile

      -f dockerfiles/Dockerfile --target prod

      -t remote-data-broker .

    - >

      docker run --rm remote-data-broker bash -c 

      "set -e;

      poetry run isort --filter-files --check-only --profile black . ;

      poetry run black --check --diff .;

      poetry run flake8 .;

      poetry run pylint remote_data_broker tests;

      poetry run mypy .

      "

    - docker tag remote-data-broker $CONTAINER_RDM_URL/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

    - docker push $CONTAINER_RDM_URL/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

    - docker tag remote-data-broker $CONTAINER_RDM_URL:$CI_COMMIT_SHORT_SHA

    - docker push $CONTAINER_RDM_URL:$CI_COMMIT_SHORT_SHA

  tags:

    - rse-multi-builder



service-tests:

  stage: unit-tests

  variables:

  script:

    docker run --rm

    -v $PWD:/reports

    $CONTAINER_RDM_URL/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA bash -c "poetry run pytest tests/unit --junitxml=/reports/report_ut.xml"

  artifacts:

    when: always

    reports:

      junit: report_ut.xml

  tags:

    - rse-multi-builder



service-coverage:

  stage: unit-tests

unit-tests:

  stage: test

  variables:

  script:

    - >

      docker run --rm

      -v $PWD:/reports

      $CONTAINER_RDM_URL/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA bash -c "poetry run pytest --cov=. --cov-fail-under=75  --cov-report=html:/reports/htmlcov --cov-report=xml:/reports/coverage.xml tests/unit "

    - sed -i "s:<source>/code:<source>${CI_BUILDS_DIR}/${CI_PROJECT_PATH}:g" coverage.xml

  coverage: '/coverage: \d+.\d+%/'

  artifacts:

    reports:

      coverage_report:

        coverage_format: cobertura

        path: cobertura.xml

    paths:

      - htmlcov/

  tags:

    - rse-multi-builder



service-build:

  stage: build-prod

    RABBITMQ_HOST: "rabbitmq"

  script:

    - >

      docker build

      -f dockerfiles/Dockerfile --target production

      -f dockerfiles/Dockerfile --target package

      -t remote-data-broker .

    - docker tag remote-data-broker $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

    - docker push $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

  tags:

    - rse-multi-builder







service-integration-tests:

  stage: integration-tests

  variables:

    LOCALHOST: "172.17.0.1"

  script:

    - >

      docker run --rm -d

      --name remote-data-broker

      -p 8090:8000

      $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

    - sleep 2

    - >

      docker run --rm

      -e LOCALHOST

      -v $PWD:/reports

      $CONTAINER_RDM_URL/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA bash -c "poetry run pytest tests/integration --junitxml=/reports/report_int.xml"

    - docker rm -f remote-data-broker

      --name $RABBITMQ_HOST

      -p 5672:5672

      code.ornl.gov:4567/rse/images/rabbitmq:3.9.27-management

    - sleep 10

    - docker logs $RABBITMQ_HOST

    - docker run --network="host" --rm -v `pwd`:/tmp remote-data-broker bash -c "cd src && go test -v ./... > out && cat out && cat out | go-junit-report -set-exit-code > /tmp/report.xml"

    - docker rm -f $RABBITMQ_HOST

  artifacts:

    when: always

    paths:

      - report.xml

    reports:

      junit: report_int.xml

      junit: report.xml

  tags:

    - rse-multi-builder



update-latest-tag:

  stage: update-repo



image-tag:

  stage: build-prod

  script:

    - docker pull $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

    - docker tag $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:latest

    - docker push $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:latest

    - docker pull $CONTAINER_RDM_URL:$CI_COMMIT_SHORT_SHA

    - docker run --rm -v `pwd`:/tmp remote-data-broker bash -c "cp /rdb/build/*.deb /tmp"

  when: manual

  tags:

    - rse-multi-builder



package:

  stage:     update-repo

build-packages:

  stage: build-prod

  script:

    - >

      docker run

      $CONTAINER_RDM_URL_PROD/$CI_COMMIT_REF_NAME:$CI_COMMIT_SHORT_SHA

      bash -c "poetry build && 

      poetry config repositories.gitlab_repo ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi && 

      poetry publish -u gitlab-ci-token -p ${CI_JOB_TOKEN} -r gitlab_repo

      "

    - docker build -f dockerfiles/Dockerfile -t remote-data-broker --target package .

    - docker run --rm -v `pwd`:/tmp remote-data-broker bash -c "cp /rdb/build/*.deb /tmp"

    - fname=`ls *.deb | head -n 1`

    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file $fname "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/deb/${TAG}/$fname"'

  when: manual

  tags:

    - rse-multi-builder

.pre-commit-config.yaml

deleted100644 → 0
+0 −73
Original line number Diff line number Diff line 
repos:

  - repo: local

    hooks:

      - id: poetry-check

        name: Poetry Check

        description: run poetry check to validate config

        entry: poetry check

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: poetry-lock

        name: Poetry Lock

        description: Refresh, but do not update, the Poetry lock file.

        entry: poetry lock --no-update --no-interaction

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Black

        description: use Black for formatting

        entry: poetry run black .

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Isort

        description: use Isort to sort imports

        entry: poetry run isort --filter-files --profile black .

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Flake8

        description: use Flake8 to lint

        entry: poetry run flake8

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Pylint

        description: use pylint to lint

        entry: poetry run pylint remote_data_broker tests

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Mypy

        description: use mypy to check types

        entry: poetry run mypy .

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Unit tests

        description: use pytest to run unit tests

        entry: poetry run pytest tests/unit

        pass_filenames: false

        language: system

  - repo: local

    hooks:

      - id: system

        name: Coverage

        description: use pytest to run unit tests with coverage

        entry: poetry run pytest  --cov=. --cov-fail-under=75 tests/unit

        pass_filenames: false

        language: system
 
 No newline at end of file