Merge pull request #3 from cisagov/first-commits

Initial Functionality

Author: mcdonnnj

.github/lineage.yml

@@ -3,4 +3,4 @@ version: "1"
 
 lineage:
   skeleton:
-    remote-url: https://github.com/cisagov/skeleton-generic.git
+    remote-url: https://github.com/cisagov/skeleton-docker.git

.github/workflows/build.yml

@@ -26,10 +26,9 @@ on:
 
 env:
   BUILDX_CACHE_DIR: ~/.cache/buildx
-  IMAGE_NAME: cisagov/example
+  IMAGE_NAME: cisagov/vdp-scanner
   PIP_CACHE_DIR: ~/.cache/pip
-  PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\
-  linux/arm64,linux/ppc64le,linux/s390x"
+  PLATFORMS: "linux/amd64,linux/arm/v7,linux/arm64"
   PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit
 
 jobs:

.isort.cfg

@@ -6,5 +6,7 @@ import_heading_stdlib=Standard Python Libraries
 import_heading_thirdparty=Third-Party Libraries
 import_heading_firstparty=cisagov Libraries
 
+known_first_party=hash_http_content
+
 # Run isort under the black profile to align with our other Python linting
 profile=black

CONTRIBUTING.md

@@ -15,7 +15,7 @@ all of which should be in this repository.
 
 If you want to report a bug or request a new feature, the most direct
 method is to [create an
-issue](https://github.com/cisagov/skeleton-docker/issues) in this
+issue](https://github.com/cisagov/vdp-scanner-docker/issues) in this
 repository.  We recommend that you first search through existing
 issues (both open and closed) to check if your particular issue has
 already been reported.  If it has then you might want to add a comment
@@ -25,7 +25,7 @@ one.
 ## Pull requests ##
 
 If you choose to [submit a pull
-request](https://github.com/cisagov/skeleton-docker/pulls), you will
+request](https://github.com/cisagov/vdp-scanner-docker/pulls), you will
 notice that our continuous integration (CI) system runs a fairly
 extensive set of linters and syntax checkers.  Your pull request may
 fail these checks, and that's OK.  If you want you can stop there and
@@ -111,9 +111,9 @@ can create and configure the Python virtual environment with these
 commands:
 
 ```console
-cd skeleton-docker
-pyenv virtualenv <python_version_to_use> skeleton-docker
-pyenv local skeleton-docker
+cd vdp-scanner-docker
+pyenv virtualenv <python_version_to_use> vdp-scanner-docker
+pyenv local vdp-scanner-docker
 pip install --requirement requirements-dev.txt
 ```
 

Dockerfile

@@ -1,38 +1,77 @@
-ARG VERSION=unspecified
+ARG PY_VERSION=3.9
 
-FROM python:3.9-alpine
-
-ARG VERSION
+FROM python:${PY_VERSION} AS compile-stage
 
 # For a list of pre-defined annotation keys and value types see:
 # https://github.com/opencontainers/image-spec/blob/master/annotations.md
 # Note: Additional labels are added by the build workflow.
-LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov"
+LABEL org.opencontainers.image.authors="nicholas.mcdonnell@cisa.dhs.gov"
 LABEL org.opencontainers.image.vendor="Cyber and Infrastructure Security Agency"
 
-ARG CISA_UID=421
-ENV CISA_HOME="/home/cisa"
-ENV ECHO_MESSAGE="Hello World from Dockerfile"
+RUN apt-get update \
+  && apt-get install -y --allow-downgrades --no-install-recommends \
+    libxml2-dev=2.9.4+dfsg1-7+deb10u1 \
+    libxslt1-dev=1.1.32-2.2~deb10u1
+
+ENV PY_VENV=/.venv
+
+# Manually set up the virtual environment
+RUN python -m venv --system-site-packages ${PY_VENV}
+ENV PATH="${PY_VENV}/bin:$PATH"
+
+# Install core Python dependencies
+RUN python -m pip install --no-cache-dir \
+  pip==21.0.1 \
+  pipenv==2020.11.15 \
+  setuptools==53.0.0 \
+  wheel==0.36.2
+
+# Install vdp_scanner.py requirements
+COPY src/Pipfile Pipfile
+COPY src/Pipfile.lock Pipfile.lock
+# PIPENV_VENV_IN_PROJECT=1 directs pipenv to use the current directory for venvs
+RUN PIPENV_VENV_IN_PROJECT=1 pipenv sync
+
+# We only need pipenv to set up the environment, so we remove it from the venv
+# as a last step.
+RUN python -m pip uninstall --yes pipenv
+
+FROM python:${PY_VERSION}-slim AS build-stage
+
+ARG SERVERLESS_CHROME_VERSION="v1.0.0-57"
+ARG SERVERLESS_CHROME_LOCAL="/usr/local/bin/serverless-chrome"
+
+RUN apt-get update \
+  && apt-get install -y --allow-downgrades --no-install-recommends \
+    ca-certificates=20200601~deb10u2 \
+    chromium-common=88.0.4324.182-1~deb10u1 \
+    curl=7.64.0-4+deb10u2 \
+    libnss3=2:3.42.1-1+deb10u3 \
+    libxml2-dev=2.9.4+dfsg1-7+deb10u1 \
+    libxslt1-dev=1.1.32-2.2~deb10u1 \
+    openssl=1.1.1d-0+deb10u6 \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
 
-RUN addgroup --system --gid ${CISA_UID} cisa \
-  && adduser --system --uid ${CISA_UID} --ingroup cisa cisa
+# Download the specified serverless chrome release and install it for use
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# Follow redirects and output as the specified file name
+RUN curl -L \
+  https://github.com/adieuadieu/serverless-chrome/releases/download/${SERVERLESS_CHROME_VERSION}/stable-headless-chromium-amazonlinux-2.zip \
+  | gunzip --stdout - > ${SERVERLESS_CHROME_LOCAL}
+RUN chmod 755 ${SERVERLESS_CHROME_LOCAL}
 
-RUN apk --update --no-cache add \
-ca-certificates \
-openssl \
-py-pip
+ENV PY_VENV=/.venv
+COPY --from=compile-stage ${PY_VENV} ${PY_VENV}
+ENV PATH="${PY_VENV}/bin:$PATH"
 
-WORKDIR ${CISA_HOME}
+ENV TASK_HOME="/task"
 
-RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \
-  tar xzf sourcecode.tgz --strip-components=1 && \
-  pip install --requirement requirements.txt && \
-  ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \
-  rm sourcecode.tgz
+WORKDIR ${TASK_HOME}
+RUN mkdir host_mount
 
-USER cisa
+COPY src/version.txt version.txt
+COPY src/vdp_scanner.py vdp_scanner.py
 
-EXPOSE 8080/TCP
-VOLUME ["/var/log"]
-ENTRYPOINT ["example"]
-CMD ["--log-level", "DEBUG"]
+ENTRYPOINT ["python", "vdp_scanner.py"]
+CMD ["github"]

README.md

@@ -1,72 +1,56 @@
-# skeleton-docker 💀🐳 #
+# vdp-scanner-docker 🔍📄 #
 
-[![GitHub Build Status](https://github.com/cisagov/skeleton-docker/workflows/build/badge.svg)](https://github.com/cisagov/skeleton-docker/actions)
-[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/alerts/)
-[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/skeleton-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/skeleton-docker/context:python)
+[![GitHub Build Status](https://github.com/cisagov/vdp-scanner-docker/workflows/build/badge.svg)](https://github.com/cisagov/vdp-scanner-docker/actions)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/cisagov/vdp-scanner-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/vdp-scanner-docker/alerts/)
+[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/cisagov/vdp-scanner-docker.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/cisagov/vdp-scanner-docker/context:python)
 
 ## Docker Image ##
 
-[![Docker Pulls](https://img.shields.io/docker/pulls/cisagov/example)](https://hub.docker.com/r/cisagov/example)
-[![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/example)](https://hub.docker.com/r/cisagov/example)
-[![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/skeleton-docker/tags)
+[![Docker Pulls](https://img.shields.io/docker/pulls/cisagov/vdp-scanner-docker)](https://hub.docker.com/r/cisagov/vdp-scanner)
+[![Docker Image Size (latest by date)](https://img.shields.io/docker/image-size/cisagov/vdp-scanner)](https://hub.docker.com/r/cisagov/vdp-scanner)
+[![Platforms](https://img.shields.io/badge/platforms-amd64%20%7C%20arm%2Fv6%20%7C%20arm%2Fv7%20%7C%20arm64%20%7C%20ppc64le%20%7C%20s390x-blue)](https://hub.docker.com/r/cisagov/vdp-scanner/tags)
 
-This is a docker skeleton project that can be used to quickly get a
-new [cisagov](https://github.com/cisagov) GitHub docker project
-started.  This skeleton project contains [licensing
-information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com)
-and [GitHub Actions](https://github.com/features/actions) configurations
-appropriate for docker containers and the major languages that we use.
+This is a Docker project to scan either the
+[GSA current Federal .gov domain list](https://github.com/GSA/data/blob/master/dotgov-domains/current-federal.csv)
+or a given CSV in the same format with the
+[cisagov/hash-http-content](https://github.com/cisagov/hash-http-content)
+Python library. Then it will output CSVs with agency and domain level results.
 
 ## Usage ##
 
 ### Install ###
 
-Pull `cisagov/example` from the Docker repository:
+Pull `cisagov/vdp-scanner` from the Docker repository:
 
-    docker pull cisagov/example
+```console
+docker pull cisagov/vdp-scanner
+```
 
-Or build `cisagov/example` from source:
+Or build `cisagov/vdp-scanner` from source:
 
-    git clone https://github.com/cisagov/skeleton-docker.git
-    cd skeleton-docker
-    docker-compose build --build-arg VERSION=0.0.1
+```console
+git clone https://github.com/cisagov/vdp-scanner-docker.git
+cd vdp-scanner-docker
+docker-compose build
+```
 
 ### Run ###
 
-    docker-compose run --rm example
+This Docker image needs a bind mount to get the output from the script to the
+host.
 
-## Ports ##
+Using `docker run`
 
-This container exposes the following ports:
+```console
+docker run --mount type=bind,source=$(pwd),target=/task/host_mount --rm cisagov/vdp-scanner
+```
 
-| Port  | Protocol | Service  |
-|-------|----------|----------|
-| 8080  | TCP      | http     |
+or if you have cloned the repository, you can use the included
+`docker-compose.yml`
 
-## Environment Variables ##
-
-| Variable      | Default Value                 | Purpose      |
-|---------------|-------------------------------|--------------|
-| ECHO_MESSAGE  | `Hello World from Dockerfile` | Text to echo |
-
-## Secrets ##
-
-| Filename      | Purpose              |
-|---------------|----------------------|
-| quote.txt     | Secret text to echo  |
-
-## Volumes ##
-
-| Mount point | Purpose        |
-|-------------|----------------|
-| /var/log    | logging output |
-
-## New Repositories from a Skeleton ##
-
-Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup)
-for step-by-step instructions on how to start a new repository from
-a skeleton. This will save you time and effort when configuring a
-new repository!
+```console
+docker-compose up
+```
 
 ## Contributing ##
 

docker-compose.yml

@@ -3,35 +3,25 @@ version: "3.7"
 
 # This docker-compose file is used to build and test the container
 
-secrets:
-  quote_txt:
-    file: ./src/secrets/quote.txt
-
 services:
-  example:
+  vdp-scanner:
     # Run the container normally
     build:
-      # VERSION must be specified on the command line:
-      # e.g., --build-arg VERSION=0.0.1
+      # SERVERLESS_CHROME_VERSION and SERVERLESS_CHROME_LOCAL can be specified
+      # on the command line to modify what is installed and where:
+      # --build-arg SERVERLESS_CHROME_VERSION=v1.0.0-56
+      # --build-arg SERVERLESS_CHROME_LOCAL=/opt/serverless-chrome
       context: .
       dockerfile: Dockerfile
-    image: cisagov/example
+    image: cisagov/vdp-scanner
     init: true
     restart: "no"
-    environment:
-      - ECHO_MESSAGE=Hello World from docker-compose!
-    ports:
-      - target: "8080"
-        published: "8080"
-        protocol: tcp
-        mode: host
-    secrets:
-      - source: quote_txt
-        target: quote.txt
+    volumes:
+      - .:/task/host_mount
 
-  example-version:
+  vdp-scanner-version:
     # Run the container to collect version information
-    image: cisagov/example
+    image: cisagov/vdp-scanner
     init: true
     restart: "no"
     command: --version

src/Pipfile

@@ -0,0 +1,18 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+docopt = "*"
+hash-http-content = {file = "https://github.com/cisagov/hash-http-content/archive/v0.0.1.tar.gz"}
+requests = "*"
+urllib3 = "*"
+pip = "*"
+setuptools = "*"
+wheel = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3"