From 3442047c915b3a50a4024d627787fd8d13cab9ac Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Fri, 1 Dec 2023 21:43:42 -0600 Subject: [PATCH 01/13] Fix dev container --- .devcontainer/Dockerfile | 12 ------- .devcontainer/devcontainer.json | 54 +++++++++++++++---------------- .devcontainer/docker-compose.yaml | 10 ------ LNX-docker-compose.yml | 8 ++--- 4 files changed, 31 insertions(+), 53 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/docker-compose.yaml diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index e008c9287..000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -# Note: You can use any Debian/Ubuntu based image you want. -FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye - -RUN \ - apt update && \ - apt-get install bash-completion graphviz default-mysql-client -y && \ - pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \ - pip uninstall datajoint -y - -ENV DJ_HOST fakeservices.datajoint.io -ENV DJ_USER root -ENV DJ_PASS simple \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a5db4d4c5..08a4482cf 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,34 +1,34 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose { - "name": "Development", - "dockerComposeFile": "docker-compose.yaml", + "name": "Existing Docker Compose (Extend)", + // Update the 'dockerComposeFile' list if you have more compose files or use different names. + // The .devcontainer/docker-compose.yml file contains any overrides you need/want to make. + "dockerComposeFile": [ + "../LNX-docker-compose.yml", + "docker-compose.yml" + ], + // The 'service' property is the name of the service for the container that VS Code should + // use. Update this value and .devcontainer/docker-compose.yml to the real service name. "service": "app", + // The optional 'workspaceFolder' property is the path VS Code should open by default when + // connected. This is typically a file mount in .devcontainer/docker-compose.yml "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", - // Use this environment variable if you need to bind mount your local source code into a new container. - "remoteEnv": { - "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" - }, - // https://containers.dev/features + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + // Uncomment the next line if you want start specific services in your Docker Compose config. + // "runServices": [], + // Uncomment the next line if you want to keep your containers running after VS Code shuts down. + "shutdownAction": "stopCompose", + // Uncomment the next line to run commands after the container is created. + "postCreateCommand": "python3 -m pip install -e .", "features": { - "ghcr.io/devcontainers/features/docker-in-docker:2": {}, - "ghcr.io/devcontainers/features/git:1": {}, - "ghcr.io/eitsupi/devcontainer-features/jq-likes:1": {}, - "ghcr.io/guiyomh/features/vim:0": {} + "ghcr.io/cirolosapio/devcontainers-features/alpine-git:0": {}, }, - "onCreateCommand": "pip install -e .", - "postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait", - "forwardPorts": [ - 80, - 443, - 3306, - 8080, - 9000 - ], - "customizations": { - "vscode": { - "extensions": [ - "ms-python.python" - ] - } - } + // Configure tool-specific properties. + // "customizations": {}, + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" } \ No newline at end of file diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml deleted file mode 100644 index a456ed151..000000000 --- a/.devcontainer/docker-compose.yaml +++ /dev/null @@ -1,10 +0,0 @@ -version: "3" -services: - app: - build: . - extra_hosts: - - fakeservices.datajoint.io:127.0.0.1 - volumes: - - ../..:/workspaces:cached - entrypoint: /usr/local/share/docker-init.sh - command: tail -f /dev/null diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 9c0a95b78..248b3611c 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -7,7 +7,7 @@ x-net: services: db: <<: *net - image: datajoint/mysql:${MYSQL_VER} + image: datajoint/mysql:${MYSQL_VER:-5.7} environment: - MYSQL_ROOT_PASSWORD=${DJ_PASS} # ports: @@ -21,7 +21,7 @@ services: interval: 15s minio: <<: *net - image: minio/minio:${MINIO_VER} + image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z} environment: - MINIO_ACCESS_KEY=datajoint - MINIO_SECRET_KEY=datajoint @@ -58,7 +58,7 @@ services: # - "3306:3306" app: <<: *net - image: datajoint/djtest:py${PY_VER}-${DISTRO} + image: datajoint/djtest:py${PY_VER:-3.8}-${DISTRO:-alpine} depends_on: db: condition: service_healthy @@ -93,7 +93,7 @@ services: nosetests -vsw tests_old --with-coverage --cover-package=datajoint # ports: # - "8888:8888" - user: ${HOST_UID}:anaconda + user: ${HOST_UID:-1000}:anaconda volumes: - .:/src - /tmp/.X11-unix:/tmp/.X11-unix:rw From 6c36bd362368dc9f92a89543b97bbd1aa51afc5f Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 10:08:01 -0600 Subject: [PATCH 02/13] Add .devcontainer/docker-compose.yml --- .devcontainer/docker-compose.yml | 26 ++++++++++++++++++++++++++ .gitignore | 4 +++- 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .devcontainer/docker-compose.yml diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000..373751880 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,26 @@ +version: '2.4' +services: + # Update this to the name of the service you want to work with in your docker-compose.yml file + app: + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" + # array). The sample below assumes your primary file is in the root of your project. + # + # build: + # context: . + # dockerfile: .devcontainer/Dockerfile + + volumes: + # Update this to wherever you want VS Code to mount the folder of your project + - ..:/workspaces:cached + + # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust. + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + + # Overrides default command so things don't shut down after the process ends. + command: /bin/sh -c "while sleep 1000; do :; done" + diff --git a/.gitignore b/.gitignore index 4b7bdb2c4..6e1d664ff 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,6 @@ docs/site !.vscode/settings.json !.vscode/launch.json -!.devcontainer/devcontainer.json \ No newline at end of file +!.devcontainer/devcontainer.json +!.devcontainer/docker-compose.yml + From c6ccaa3e296d4728f0fbb99dd9f11e8ca209b262 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 11:26:18 -0600 Subject: [PATCH 03/13] Set default MYSQL_PASS env var --- LNX-docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index 248b3611c..f90e73a04 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -9,7 +9,7 @@ services: <<: *net image: datajoint/mysql:${MYSQL_VER:-5.7} environment: - - MYSQL_ROOT_PASSWORD=${DJ_PASS} + - MYSQL_ROOT_PASSWORD=${DJ_PASS:-simple} # ports: # - "3306:3306" # volumes: @@ -69,7 +69,7 @@ services: environment: - DJ_HOST=fakeservices.datajoint.io - DJ_USER=root - - DJ_PASS + - DJ_PASS=simple - DJ_TEST_HOST=fakeservices.datajoint.io - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint From f72d3d1c8bf0f361673889f15dce1c844fb789ad Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 12:37:19 -0600 Subject: [PATCH 04/13] Remove git feature from dev container --- .devcontainer/devcontainer.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 08a4482cf..6e414e8b6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,11 +24,9 @@ "shutdownAction": "stopCompose", // Uncomment the next line to run commands after the container is created. "postCreateCommand": "python3 -m pip install -e .", - "features": { - "ghcr.io/cirolosapio/devcontainers-features/alpine-git:0": {}, - }, + "features": {}, // Configure tool-specific properties. // "customizations": {}, // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "devcontainer" -} \ No newline at end of file +} From cfae0db75c760e58b5b8e5e8ca967be0c633f84f Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 12:38:00 -0600 Subject: [PATCH 05/13] Update compose stack to use MySQL 8.0 by default --- LNX-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index f90e73a04..f650b81ce 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -7,7 +7,7 @@ x-net: services: db: <<: *net - image: datajoint/mysql:${MYSQL_VER:-5.7} + image: datajoint/mysql:${MYSQL_VER:-8.0} environment: - MYSQL_ROOT_PASSWORD=${DJ_PASS:-simple} # ports: From 2d588f27951e48fde141566866ea142a58eabd52 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 13:15:13 -0600 Subject: [PATCH 06/13] Readd git feature to dev container --- .devcontainer/devcontainer.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6e414e8b6..4dde61560 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -24,7 +24,9 @@ "shutdownAction": "stopCompose", // Uncomment the next line to run commands after the container is created. "postCreateCommand": "python3 -m pip install -e .", - "features": {}, + "features": { + "ghcr.io/cirolosapio/devcontainers-features/alpine-git:0": {}, + }, // Configure tool-specific properties. // "customizations": {}, // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. From 53c17aee8762289c74aff1f5db718c4eca56955d Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 13:15:30 -0600 Subject: [PATCH 07/13] Update nginx version in compose stack --- LNX-docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index f650b81ce..d8bee4456 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -44,7 +44,7 @@ services: interval: 15s fakeservices.datajoint.io: <<: *net - image: datajoint/nginx:v0.2.6 + image: datajoint/nginx:v0.2.7 environment: - ADD_db_TYPE=DATABASE - ADD_db_ENDPOINT=db:3306 From 49398539279823f1d060d0597ae2a4435c94a28a Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 13:16:14 -0600 Subject: [PATCH 08/13] Update MySQL default auth plugin https://stackoverflow.com/questions/49019652/not-able-to-connect-to-mysql-docker-from-local --- LNX-docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index d8bee4456..c669a9d5f 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -10,6 +10,7 @@ services: image: datajoint/mysql:${MYSQL_VER:-8.0} environment: - MYSQL_ROOT_PASSWORD=${DJ_PASS:-simple} + command: mysqld --default-authentication-plugin=mysql_native_password # ports: # - "3306:3306" # volumes: From 22288078b137217b2746d1c8552588a3d385b218 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 14:50:25 -0600 Subject: [PATCH 09/13] Change default MySQL root password MySQL 8.0 in Docker was ignoring the MYSQL_ROOT_PASSWORD environment variable, setting the default user/pass to root and password. Update the env vars in the dev stack to match these values. --- LNX-docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index c669a9d5f..eaf3a48cd 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -9,7 +9,7 @@ services: <<: *net image: datajoint/mysql:${MYSQL_VER:-8.0} environment: - - MYSQL_ROOT_PASSWORD=${DJ_PASS:-simple} + - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password} command: mysqld --default-authentication-plugin=mysql_native_password # ports: # - "3306:3306" @@ -70,7 +70,7 @@ services: environment: - DJ_HOST=fakeservices.datajoint.io - DJ_USER=root - - DJ_PASS=simple + - DJ_PASS=password - DJ_TEST_HOST=fakeservices.datajoint.io - DJ_TEST_USER=datajoint - DJ_TEST_PASSWORD=datajoint From 92f6d6e7a9f9e8595c1c597e10e113197c223e54 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 16:16:21 -0600 Subject: [PATCH 10/13] Add nosetests and mysql to dev container --- .devcontainer/Dockerfile | 13 +++++++++++++ .devcontainer/devcontainer.json | 26 ++++++++++++++++++++------ .devcontainer/docker-compose.yml | 18 +++++++++++------- 3 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 .devcontainer/Dockerfile diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..06958f6cd --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,13 @@ +ARG PY_VER +ARG DISTRO +FROM mcr.microsoft.com/devcontainers/python:${PY_VER}-${DISTRO} +RUN \ + apt update && \ + apt-get install bash-completion graphviz default-mysql-client -y && \ + pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \ + pip uninstall datajoint -y + +USER root +ENV DJ_HOST fakeservices.datajoint.io +ENV DJ_USER root +ENV DJ_PASS password diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4dde61560..35b1d0835 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,18 +17,32 @@ // Features to add to the dev container. More info: https://containers.dev/features. // "features": {}, // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + "forwardPorts": [ + 80, + 443, + 3306, + 8080, + 9000 + ], // Uncomment the next line if you want start specific services in your Docker Compose config. // "runServices": [], // Uncomment the next line if you want to keep your containers running after VS Code shuts down. "shutdownAction": "stopCompose", - // Uncomment the next line to run commands after the container is created. - "postCreateCommand": "python3 -m pip install -e .", + "onCreateCommand": "python3 -m pip install -e .", "features": { - "ghcr.io/cirolosapio/devcontainers-features/alpine-git:0": {}, + "ghcr.io/devcontainers/features/git:1": {}, }, // Configure tool-specific properties. - // "customizations": {}, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python" + ] + } + }, + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + } // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "devcontainer" -} +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 373751880..75c199e7a 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -2,14 +2,16 @@ version: '2.4' services: # Update this to the name of the service you want to work with in your docker-compose.yml file app: - # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer - # folder. Note that the path of the Dockerfile and context is relative to the *primary* + # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer + # folder. Note that the path of the Dockerfile and context is relative to the *primary* # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" # array). The sample below assumes your primary file is in the root of your project. - # - # build: - # context: . - # dockerfile: .devcontainer/Dockerfile + build: + context: . + dockerfile: .devcontainer/Dockerfile + args: + - PY_VER=${PY_VER:-3.8} + - DISTRO=${DISTRO:-buster} volumes: # Update this to wherever you want VS Code to mount the folder of your project @@ -21,6 +23,8 @@ services: # security_opt: # - seccomp:unconfined + user: root + # Overrides default command so things don't shut down after the process ends. command: /bin/sh -c "while sleep 1000; do :; done" - + From 39c42d05983e68a2000835d8a35e9430e40f43e2 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 21:58:43 -0600 Subject: [PATCH 11/13] Change default MySQL root password in CI --- .github/workflows/development.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/development.yaml b/.github/workflows/development.yaml index fdcbcd677..5f3b1f075 100644 --- a/.github/workflows/development.yaml +++ b/.github/workflows/development.yaml @@ -77,7 +77,7 @@ jobs: - name: Run primary tests env: PY_VER: ${{matrix.py_ver}} - DJ_PASS: simple + DJ_PASS: password MYSQL_VER: ${{matrix.mysql_ver}} DISTRO: alpine MINIO_VER: RELEASE.2021-09-03T03-56-13Z From 09a5439e074eec56a78aae88e5c186494da2b104 Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 22:06:34 -0600 Subject: [PATCH 12/13] Update developer docs with MySQL password --- docs/src/develop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/develop.md b/docs/src/develop.md index 4acb9ed35..842c04d96 100644 --- a/docs/src/develop.md +++ b/docs/src/develop.md @@ -104,7 +104,7 @@ It is often useful in development to connect to DataJoint's relational database Connect as follows to the database running within your developer environment: ``` -mysql -hfakeservices.datajoint.io -uroot -psimple +mysql -hfakeservices.datajoint.io -uroot -ppassword ``` ### Documentation From 4b7a68c2db79a2666029f88dc22dc7dcc50e974a Mon Sep 17 00:00:00 2001 From: Ethan Ho Date: Mon, 4 Dec 2023 22:54:28 -0600 Subject: [PATCH 13/13] Unique tag for devcontainer --- .devcontainer/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 75c199e7a..1ee8ea148 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -6,6 +6,8 @@ services: # folder. Note that the path of the Dockerfile and context is relative to the *primary* # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile" # array). The sample below assumes your primary file is in the root of your project. + container_name: devcontainer + image: devcontainer build: context: . dockerfile: .devcontainer/Dockerfile