From 19b4fd520a511916b5f2d33dfb31f04ceea5f2d1 Mon Sep 17 00:00:00 2001 From: Valentin Tolmer Date: Thu, 15 Jun 2023 10:24:43 +0200 Subject: [PATCH] scripts: don't export the default attribute schemas --- .github/workflows/docker-build-static.yml | 26 ++++++++++++----------- docs/database_migration.md | 13 ++++++------ scripts/sqlite_dump_commands.sh | 8 ++++++- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.github/workflows/docker-build-static.yml b/.github/workflows/docker-build-static.yml index dcb9832..c2cbc63 100644 --- a/.github/workflows/docker-build-static.yml +++ b/.github/workflows/docker-build-static.yml @@ -299,6 +299,11 @@ jobs: steps: + - name: Checkout scripts + uses: actions/checkout@v3.5.3 + with: + sparse-checkout: 'scripts' + - name: Download LLDAP artifacts uses: actions/download-artifact@v3 with: @@ -347,9 +352,7 @@ jobs: - name: Export and Converting to Postgress run: | - curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh - chmod +x ./helper.sh - ./helper.sh | sqlite3 ./users.db > ./dump.sql + bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e ":a; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),1([^']*\);)$/\1,true\2/; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),0([^']*\);)$/\1,false\2/; ta" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql - name: Create schema on postgres @@ -359,14 +362,13 @@ jobs: - name: Copy converted db to postgress and import run: | docker cp ./dump.sql postgresql:/tmp/dump.sql - docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql" + docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql" | tee import.log rm ./dump.sql + ! grep ERROR import.log > /dev/null - name: Export and Converting to mariadb run: | - curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh - chmod +x ./helper.sh - ./helper.sh | sqlite3 ./users.db > ./dump.sql + bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql cp ./dump.sql ./dump-no-sed.sql sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql @@ -377,14 +379,13 @@ jobs: - name: Copy converted db to mariadb and import run: | docker cp ./dump.sql mariadb:/tmp/dump.sql - docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql" + docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql" | tee import.log rm ./dump.sql + ! grep ERROR import.log > /dev/null - name: Export and Converting to mysql run: | - curl -L https://raw.githubusercontent.com/lldap/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh - chmod +x ./helper.sh - ./helper.sh | sqlite3 ./users.db > ./dump.sql + bash ./scripts/sqlite_dump_commands.sh | sqlite3 ./users.db > ./dump.sql sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql @@ -394,8 +395,9 @@ jobs: - name: Copy converted db to mysql and import run: | docker cp ./dump.sql mysql:/tmp/dump.sql - docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql" + docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql" | tee import.log rm ./dump.sql + ! grep ERROR import.log > /dev/null - name: Run lldap with postgres DB and healthcheck again run: | diff --git a/docs/database_migration.md b/docs/database_migration.md index 5ad5954..7973513 100644 --- a/docs/database_migration.md +++ b/docs/database_migration.md @@ -20,7 +20,7 @@ LLDAP has a command that will connect to a target database and initialize the schema. If running with docker, run the following command to use your active instance (this has the benefit of ensuring your container has access): -``` +```sh docker exec -it /app/lldap create_schema -d ``` @@ -34,7 +34,7 @@ databases (SQLite in this example) will give an error if LLDAP is in the middle statements. There are various ways to do this, but a simple enough way is filtering a whole database dump. This repo contains [a script](/scripts/sqlite_dump_commands.sh) to generate SQLite commands for creating an appropriate dump: -``` +```sh ./sqlite_dump_commands.sh | sqlite3 /path/to/lldap/config/users.db > /path/to/dump.sql ``` @@ -49,8 +49,9 @@ a transaction in case one of the statements fail. PostgreSQL uses a different hex string format. The command below should switch SQLite format to PostgreSQL format, and wrap it all in a transaction: -``` +```sh sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" \ +-e ":a; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),1([^']*\);)$/\1,true\2/; s/(INSERT INTO user_attribute_schema\(.*\) VALUES\(.*),0([^']*\);)$/\1,false\2/; ta" \ -e '1s/^/BEGIN;\n/' \ -e '$aCOMMIT;' /path/to/dump.sql ``` @@ -58,11 +59,11 @@ sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" \ ### To MySQL MySQL mostly cooperates, but it gets some errors if you don't escape the `groups` table. It also uses -backticks to escape table name instead of quotes. Run the +backticks to escape table name instead of quotes. Run the following command to wrap all table names in backticks for good measure, and wrap the inserts in a transaction: -``` +```sh sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ -e '1s/^/START TRANSACTION;\n/' \ -e '$aCOMMIT;' \ @@ -74,7 +75,7 @@ sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ While MariaDB is supposed to be identical to MySQL, it doesn't support timezone offsets on DATETIME strings. Use the following command to remove those and perform the additional MySQL sanitization: -``` +```sh sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \ -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ -e '1s/^/START TRANSACTION;\n/' \ diff --git a/scripts/sqlite_dump_commands.sh b/scripts/sqlite_dump_commands.sh index cdcf87a..a051607 100644 --- a/scripts/sqlite_dump_commands.sh +++ b/scripts/sqlite_dump_commands.sh @@ -1,9 +1,15 @@ #! /bin/bash -tables=("users" "groups" "memberships" "jwt_refresh_storage" "jwt_storage" "password_reset_tokens" "group_attribute_schema" "group_attributes" "user_attribute_schema" "user_attributes") +tables=("users" "groups" "memberships" "jwt_refresh_storage" "jwt_storage" "password_reset_tokens" "group_attribute_schema" "group_attributes") echo ".header on" for table in ${tables[@]}; do echo ".mode insert $table" echo "select * from $table;" done + +echo ".mode insert user_attribute_schema" +echo "select * from user_attribute_schema where user_attribute_schema_name not in ('first_name', 'last_name', 'avatar');" + +echo ".mode insert user_attributes" +echo "select * from user_attributes;"