spit and polish

.dockerignore

@@ -3,3 +3,4 @@
 !pyproject.toml
 !LICENSE.txt
 !README.md
+!dist/

.gitignore

 build/
 dist/
 .venv/
-.coverage/
-.mypy/
+.coverage*
+.*_cache/

.gitlab-ci.yml

+# References:
+# - Keywords: https://docs.gitlab.com/ee/ci/yaml
+# - CI Variables: https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
+# - Gitlab PyPI: https://gitlab.ssec.wisc.edu/help/user/packages/pypi_repository/index
+#
 stages:
-  - lint
+  - lint  # if lint fails, don't bother doing downstream
   - test
   - build
   - deploy
 
+# Set pip cache dir to our projects cache dir
 variables:
   PIP_CACHE_DIR: $CI_PROJECT_DIR/.cache/pip
 
+# Cache pip packages between jobs
 cache:
   paths:
     - .cache/pip
 
+# All jobs use python:3.10 image by default on the SSEC Shared CI runner
 default:
   image: python:3.10
   tags: ["docker", "ssec_shared"]
@@ -19,22 +27,25 @@ lint:
   stage: lint
   script:
     - pip install hatch
-    - hatch run lint
+    - hatch run lint:all
 
 test:unit:
   stage: test
   script:
     - pip install hatch
+    - hatch run pip list
+    - hatch run pip install -e .
     - hatch run test tests/unit
 
 test:int:
   stage: test
   script:
     - pip install hatch
+    - hatch run pip install -e .
     - hatch run test tests/int
 
-# Only for the default branch, no tags
-build:
+# Always do dist build for the default branch
+build:dist:
   stage: build
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
@@ -42,45 +53,44 @@ build:
     - pip install hatch
     - hatch build
   artifacts:
+    # Don't need to keep these long as they're just used in docker image or pushed
+    # to the package registry
+    expire_in: 1 day
     paths:
       - ./dist/*.whl
       - ./dist/*.tar.gz
 
-# All default branch commits get an image
-build:image:
+# Only build image with sha tag for the default branch commits
+build:image:dev:
   stage: build
+  # Make sure we have the dist build artifacts when building the image
+  needs: ["build:dist"]
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
   rules:
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-    - if: $CI_COMMIT_TAG =~ /\d+\.\d+\.\d+$/
-  image: docker:24.0.5
-  services:
-    - docker:24.0.5-dind
   script:
-    - docker login -u "${SIPS_REGISTRY_USER}" -p "${SIPS_REGISTRY_TOKEN}" "${CI_REGISTRY}"
-    - docker build -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}" .
-    - docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}"
-    - [[ -n "${CI_COMMIT_TAG}" ]] && 
-      docker build -t "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" . && 
-      docker push "${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}" 
-      
+    - /kaniko/executor
+      --context "$CI_PROJECT_DIR"
+      --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHORT_SHA}
 
-# Only for tagged X.X.X versions
-prod:build:image:
+# Only build image with git veresion tag for default branch tags matching X.X.X
+build:image:release:
   stage: build
-  image: docker:24.0.5
+  # Make sure we have the dist build artifacts when building the image
+  needs: ["build:dist"]
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
   rules:
     - if: $CI_COMMIT_TAG =~ /\d+\.\d+\.\d+$/
-  services:
-    - docker:24.0.5-dind
-  variables:
-    img: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
   script:
-    - docker build -t "${img}" .
-    - docker login -u "${SIPS_REGISTRY_USER}" -p "${SIPS_REGISTRY_TOKEN}" "${CI_REGISTRY}"
-    - docker push "${img}"
+    - /kaniko/executor
+      --context "$CI_PROJECT_DIR"
+      --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_TAG}
 
-# Only deploy tagged X.X.X versions
-prod:deploy:packages:
+deploy:packages:release:
   stage: deploy
   rules:
     - if: $CI_COMMIT_TAG =~ /\d+\.\d+\.\d+$/
@@ -88,5 +98,5 @@ prod:deploy:packages:
     HATCH_INDEX_REPO: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/pypi
     HATCH_INDEX_AUTH: $CI_JOB_TOKEN
   script:
-    - hatch publish -y
-
+    # -y is required because we set disable=true in [tools.hatch.publish.index]
+    - hatch publish -y -u "${CI_DEPLOY_USER}"

.pre-commit-config.yaml

+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+    - id: check-yaml
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.0.289
+  hooks:
+    - id: ruff
+      args: [--fix, --exit-non-zero-on-fix]
+- repo: https://github.com/psf/black
+  rev: 22.10.0
+  hooks:
+    - id: black

Dockerfile

-FROM python:3.10 AS base
-# ENV PIP_INDEX_URL=http://devpi.sips/sips/prod/+simple
-# ENV PIP_TRUSTED_HOST=devpi.sips
-ENV PIP_NO_CACHE_DIR="false"
-RUN apt-get update && apt-get -yqq install --no-install-recommends \
-    dumb-init \
-  && \
-  rm -rf /var/lib/apt/lists/*
-
-FROM base AS dev
-WORKDIR /code
-COPY . .
-RUN pip install hatch && \
-    hatch dep show requirements -p > project_requirements.txt && \
-    hatch dep show requirements > all_requirements.txt && \
-    hatch run python3 -m pip install -r all_requirements.txt
-RUN hatch run python3 -m pip install -e . && \
-    hatch build
-
-FROM base as app
-COPY --from=dev /code/project_requirements.txt /tmp/requirements.txt
-COPY --from=dev /code/dist/proj-*.whl /tmp/
-RUN python -m pip install -r /tmp/requirements.txt /tmp/proj-*.whl
+FROM python:3.10
+COPY ./dist/proj-*.whl /tmp/
+RUN python -m pip install /tmp/proj-*.whl
 RUN useradd app
 USER app
 ENTRYPOINT ["dumb-init", "--"]

README.md

 # Python Project, 2023 Style
 
-References:
+This is an example repository intended to demonstrate ~best~ ~good~ ~standard~
+practices for Python project.
+
+> Much of this project was put together by reading documentation from the various
+> tools that are being used. Although I do use many of these tool in my Python projects
+> I don't have a ton of miles on any specific project using all the features used with
+> the specific configurations here.
+>
+> So, there may be some issues here and there. Feel free to correct them and submit
+> a merge request or simply file an issue.
+
+> You can also feel free to submit issues or merge requests with suggestions or comments
+  regarding other useful tools and techniques that would be valuable to add.
+
+## Features
+* Project management via [hatch](https://hatch.pypa.io)
+    - Packaging
+    - Build and tooling environments
+    - Dynamic versioning using GIT tags
+* Linting and formatting
+    - [black](https://black.readthedocs.io)
+        * Formatting
+    - [ruff](https://beta.ruff.rs/docs/)
+        * Linting
+    - [mypy](https://black.readthedocs.io)
+        * Static type checker
+    - [isort](https://pycqa.github.io/isort/)
+* Testing and coverage
+    - [pytest](https://docs.pytest.org/en/7.4.x)
+    - [coverage](https://coverage.readthedocs.io/en/7.3.1/)
+* Docker image w/ rootless build
+    - [kaniko](https://github.com/GoogleContainerTools/kaniko)
+* Pre-commit GIT hooks
+    - [pre-commit](https://pre-commit.com/)
+* Gitlab CI/CD
+
+The software itself it not intended to be particularly useful.
+
+## Other References:
 * Python Packaging Authority [PyPA](https://www.pypa.io)
 * Python Package Index [PyPI](https://pypi.org)
 * [Pip](https://pip.pypa.io)
     - "… the package installer for Python"
 * [Hatch](https://hatch.pypa.io)
-    - "… modern, extensible Python project manager"​
-* [Poetry](https://python-poetry.org)
-    - "Python packaging and dependency management made easy"​
-* [Pipenv](https://pipenv.pypa.io)
-    - "… Python virutalenv management tool …"​
-* [Setuptools](https://setuptools.pypa.io)
-    - "... fully-featured, actively-maintained, and stable library designed to facilitate packaging Python projects"
+    - "… modern, extensible Python project manager"
+* [`gitlab-ci.yml`](https://docs.gitlab.com/ee/ci/yaml)
+* [CI Variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)
+* Gitlab [PyPI](https://gitlab.ssec.wisc.edu/help/user/packages/pypi_repository/index)
+
+# Dynamic Versioning
+Hatch is configured to manage the package version metadata specified using the
+`__version__` variable in `proj/__init__.py`.
+
+The `release.sh` script is provided to help increment the package major, minor, or
+patch versions and create and put an associated GIT tag for the new version.
+
+For example, if the current version is `0.1.0` and you run `./release.sh patch` then:
+1. The package version will be incremented to `0.1.1` in `proj/__init__.py`
+2. The version file `proj/__init__.py` will be committed to git with the commit message
+   "bump version" on the current branch
+3. All commits and tags from the current branch will be pushed
+   - `git push --follow-tags`
+
+# hatch
+> This configuration is the default configuration provided when creating a new hatch project
+  using `hatch new` with few modifications.
+
+The hatch configuration found in [pyproject.toml](https://gitlab.ssec.wisc.edu/brucef/pyproj/-/blob/main/pyproject.toml)
+supports a couple of different [environments](https://hatch.pypa.io/latest/environment/).
+There is the `default` environment, which is used when running any command using
+`hatch run` when not explicitly specifying the environment.
+
+
+For example, to run the `test-cov` script defined in the default envinronment, .i.e.,
+the `[tool.hatch.envs.default.scripts]` section of the configuration:
+```sh
+hatch run test-cov
+```
+
+This configuration provides `default` environment containing scripts for testing and coverage
+and the `lint` environment containing scripts for formatting and linting.
+
+To run all linters to perform checking only:
+```sh
+hatch run lint:all
+```
+
+Or to apply fixes:
+```sh
+hatch run lint:fmt
+```
+
+# CI/CD
+
+## Test Jobs
+All the test jobs run in a default `python:3.10` container. The container does not
+have hatch installed by default, so hatch is installed in each job. All jobs make use
+of the global [cache](https://docs.gitlab.com/ee/ci/yaml/#cache) and
+[variables](https://docs.gitlab.com/ee/ci/yaml/#variables) directives that will cause
+the pip cache to persist between builds.
+
+There is also a specific `lint` job to check style and formatting. It has its own
+[stage](https://docs.gitlab.com/ee/ci/yaml/#stage) defined specifically to be before
+the other test stages. This results in all downstream jobs being skipped if the lint
+job fails.
+
+## Docker Images
+This project creates a Docker Image built in CI without root permissions using
+[kaniko](https://github.com/GoogleContainerTools/kaniko). Building using kaniko is
+nice because it does not require root privileges, but I believe building this way
+precludes layer caching making the image builds much more heavy weight.
+
+And alternative would be to build using
+[Docker-in-Docker](https://docs.gitlab.com/ee/ci/docker/using_docker_build.html#use-docker-in-docker),
+however, I was not able to get this to work using the SSEC Shared Runner.
+
+Images are built and pushed for commits to the main branch tagged using the first 8
+characters of the GIT commit sha and also for any commit with a tag matching `X.X.X`.
+
+The image build job (`build:image`) relies on artifacts produced upstream by the
+`build:dist` job. This dependency is reflected by the `needs: ["build:dist"]`
+configuration, which causes these 2 jobs to run serially event though they have the
+same stage.
+
+## Python Packages
+Both the source and wheel distributions are uploaded to the project PyPI repository for
+all tagged `X.X.X` commits.
+
+See the Gitlab [PyPI](https://gitlab.ssec.wisc.edu/help/user/packages/pypi_repository/index)
+docs for how to use the repository.
+
+> **NOTE**: This project is under my Gitlab user, not a Gitlab group. When a project is in
+  a group any packages published to project repositores are also
+  [made available](https://docs.gitlab.com/ee/user/packages/pypi_repository/#install-from-the-group-level)
+  by the group level repository.

proj/__init__.py

-__version__ = "0.0.0"
+__version__ = "0.0.2"
 
 import dateparser
 
 
 def main():
-    print("Hello from proj")
+    print("Hello from proj")  # noqa: T201
     now = dateparser.parse("now")
-    print(f"The time is {now}")
+    print(f"The time is {now}")  # noqa: T201

pyproject.toml

@@ -17,12 +17,16 @@ dependencies = ["dateparser ~= 1.1"]
 [project.urls]
 Homepage = "https://gitlab.ssec.wisc.edu/brucef/pyproj"
 
+[project.scripts]
+proj = "proj:main"
+
 [tool.hatch.version]
 path = "proj/__init__.py"
 
 [tool.hatch.envs.default]
 dependencies = [
   "coverage[toml]>=6.5",
+  "pre-commit",
   "pytest",
 ]
 
@@ -115,6 +119,8 @@ ignore = [
   "FBT003",
   # Ignore checks for possible passwords
   "S105", "S106", "S107",
+  # Prone to false positives
+  "S603",
   # Ignore complexity
   "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
 ]

release.sh

@@ -11,11 +11,19 @@
 #
 set -e
 
+readonly DEFAULT_BRANCH=main
+
 if [[ ! "$1" =~ ^(major|minor|patch|dev)$ ]]; then
     echo "USAGE: $0 <major|minor|patch|dev>"
     exit 1
 fi
 
+readonly branch="$(git branch --show-current)"
+if [[ $branch != "main" ]]; then
+  echo "Releases must be done on the default branch '$DEFAULT_BRANCH', not '$branch'"
+  exit 1
+fi
+
 # Bumps the version, i.e., sets the version in the version file specified in
 # the hatch version file config.
 hatch version $1

tests/__init__.py

-# SPDX-FileCopyrightText: 2023-present Bruce Flynn <brucef@ssec.wisc.edu>
-#
-# SPDX-License-Identifier: MIT

tests/int/test_int.py

+import subprocess
+import sys
+
+
+def test_main():
+    bin_ = sys.executable.replace("/python", "/proj")
+    proc = subprocess.run([bin_], capture_output=True, check=False)
+    assert proc.returncode == 0
+    assert "Hello from proj" in proc.stdout.decode()

tests/unit/test_main.py

+import proj
+
+
+def test_main():
+    proj.main()