From 25b0a97851c143a9252594d9949f38334a5b06cb Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 12 Jun 2024 14:43:03 +0800 Subject: [PATCH] build: use Poetry as default build system for dependency installation in CI jobs (#5088) --- .github/workflows/api-tests.yml | 16 +++--- .github/workflows/db-migration-test.yml | 12 ++-- .github/workflows/style.yml | 9 ++- api/migrations/README | 1 + api/poetry.lock | 74 ++++++++++++++++++++++++- api/pyproject.toml | 10 ++++ 6 files changed, 106 insertions(+), 16 deletions(-) diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml index 3aa70420b3..e2ef85431d 100644 --- a/.github/workflows/api-tests.yml +++ b/.github/workflows/api-tests.yml @@ -14,7 +14,6 @@ concurrency: jobs: test: - name: API Tests runs-on: ubuntu-latest strategy: matrix: @@ -81,6 +80,7 @@ jobs: run: dev/pytest/pytest_vdb.sh test-in-poetry: + name: API Tests runs-on: ubuntu-latest strategy: matrix: @@ -89,26 +89,26 @@ jobs: - "3.11" steps: - - name: Install poetry - uses: abatilo/actions-poetry@v3 - with: - poetry-version: "1.8.1" - - name: Checkout code uses: actions/checkout@v4 + - name: Install Poetry + uses: abatilo/actions-poetry@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} cache: 'poetry' - cache-dependency-path: ./api/poetry.lock + cache-dependency-path: | + api/pyproject.toml + api/poetry.lock - name: Poetry check run: poetry check -C api - name: Install dependencies - run: poetry install -C api + run: poetry install -C api --with dev - name: Run Unit tests run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh diff --git a/.github/workflows/db-migration-test.yml b/.github/workflows/db-migration-test.yml index cb8dd06c5e..2f059bea9d 100644 --- a/.github/workflows/db-migration-test.yml +++ b/.github/workflows/db-migration-test.yml @@ -23,16 +23,20 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Install Poetry + uses: abatilo/actions-poetry@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: 'pip' + cache: 'poetry' cache-dependency-path: | - ./api/requirements.txt + api/pyproject.toml + api/poetry.lock - name: Install dependencies - run: pip install -r ./api/requirements.txt + run: poetry install -C api - name: Set up Middleware uses: hoverkraft-tech/compose-action@v2.0.0 @@ -50,4 +54,4 @@ jobs: - name: Run DB Migration run: | cd api - flask db upgrade + poetry run python -m flask db upgrade diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 7dad707fea..160641fdf0 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -24,6 +24,9 @@ jobs: with: files: api/** + - name: Install Poetry + uses: abatilo/actions-poetry@v3 + - name: Set up Python uses: actions/setup-python@v5 if: steps.changed-files.outputs.any_changed == 'true' @@ -32,15 +35,15 @@ jobs: - name: Python dependencies if: steps.changed-files.outputs.any_changed == 'true' - run: pip install ruff dotenv-linter + run: poetry install -C api --only lint - name: Ruff check if: steps.changed-files.outputs.any_changed == 'true' - run: ruff check --preview ./api + run: poetry run -C api ruff check --preview ./api - name: Dotenv check if: steps.changed-files.outputs.any_changed == 'true' - run: dotenv-linter ./api/.env.example ./web/.env.example + run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example - name: Lint hints if: failure() diff --git a/api/migrations/README b/api/migrations/README index 0e04844159..220678df7a 100644 --- a/api/migrations/README +++ b/api/migrations/README @@ -1 +1,2 @@ Single-database configuration for Flask. + diff --git a/api/poetry.lock b/api/poetry.lock index 7dddf47433..5819770652 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -970,6 +970,23 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "click-default-group" +version = "1.2.4" +description = "click_default_group" +optional = false +python-versions = ">=2.7" +files = [ + {file = "click_default_group-1.2.4-py2.py3-none-any.whl", hash = "sha256:9b60486923720e7fc61731bdb32b617039aba820e22e1c88766b1125592eaa5f"}, + {file = "click_default_group-1.2.4.tar.gz", hash = "sha256:eb3f3c99ec0d456ca6cd2a7f08f7d4e91771bef51b01bdd9580cc6450fe1251e"}, +] + +[package.dependencies] +click = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "click-didyoumean" version = "0.3.1" @@ -1491,6 +1508,24 @@ files = [ {file = "docstring_parser-0.16.tar.gz", hash = "sha256:538beabd0af1e2db0146b6bd3caa526c35a34d61af9fd2887f3a8a27a739aa6e"}, ] +[[package]] +name = "dotenv-linter" +version = "0.5.0" +description = "Linting dotenv files like a charm!" +optional = false +python-versions = ">=3.9,<4.0" +files = [ + {file = "dotenv_linter-0.5.0-py3-none-any.whl", hash = "sha256:fd01cca7f2140cb1710f49cbc1bf0e62397a75a6f0522d26a8b9b2331143c8bd"}, + {file = "dotenv_linter-0.5.0.tar.gz", hash = "sha256:4862a8393e5ecdfb32982f1b32dbc006fff969a7b3c8608ba7db536108beeaea"}, +] + +[package.dependencies] +attrs = "*" +click = ">=6,<9" +click_default_group = ">=1.2,<2.0" +ply = ">=3.11,<4.0" +typing_extensions = ">=4.0,<5.0" + [[package]] name = "duckdb" version = "1.0.0" @@ -4550,6 +4585,17 @@ files = [ dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "ply" +version = "3.11" +description = "Python Lex & Yacc" +optional = false +python-versions = "*" +files = [ + {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, + {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, +] + [[package]] name = "portalocker" version = "2.8.2" @@ -5818,6 +5864,32 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "ruff" +version = "0.4.8" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.4.8-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066"}, + {file = "ruff-0.4.8-py3-none-macosx_11_0_arm64.whl", hash = "sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b"}, + {file = "ruff-0.4.8-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_i686.whl", hash = "sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed"}, + {file = "ruff-0.4.8-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa"}, + {file = "ruff-0.4.8-py3-none-win32.whl", hash = "sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9"}, + {file = "ruff-0.4.8-py3-none-win_amd64.whl", hash = "sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d"}, + {file = "ruff-0.4.8-py3-none-win_arm64.whl", hash = "sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780"}, + {file = "ruff-0.4.8.tar.gz", hash = "sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268"}, +] + [[package]] name = "s3transfer" version = "0.6.2" @@ -7451,4 +7523,4 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a525ad5ee7a98124a23736c8ba5e0d1c7cf6d037a19aaac78364ae8c0ce0c242" +content-hash = "f0a99f84a0dc46b22eaea2344f205fda66f1e99be10806749492e12a7bcf66cf" diff --git a/api/pyproject.toml b/api/pyproject.toml index 9213482547..9f4c578322 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -179,6 +179,8 @@ google-cloud-aiplatform = "1.49.0" vanna = {version = "0.5.5", extras = ["postgres", "mysql", "clickhouse", "duckdb"]} kaleido = "0.2.1" +[tool.poetry.group.dev] +optional = true [tool.poetry.group.dev.dependencies] coverage = "~7.2.4" @@ -186,3 +188,11 @@ pytest = "~8.1.1" pytest-benchmark = "~4.0.0" pytest-env = "~1.1.3" pytest-mock = "~3.14.0" + +[tool.poetry.group.lint] +optional = true + +[tool.poetry.group.lint.dependencies] +ruff = "~0.4.8" +dotenv-linter = "~0.5.0" +