feat(ops): phase 8 — data migration scripts + fix password_reset_tokens in schema.sql
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 4.6
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/bin/bash
|
||||
# Usage: SSH_PASS=<password> bash db/01-load-schema.sh
|
||||
# Or: configure SSH key auth first, then just: bash db/01-load-schema.sh
|
||||
set -e
|
||||
|
||||
SERVER="setia@ims.setia.com.my"
|
||||
PORT=9321
|
||||
|
||||
echo "==> Copying schema files to server..."
|
||||
scp -P "$PORT" db/schema.sql db/rls.sql "$SERVER:/tmp/"
|
||||
|
||||
echo "==> Loading schema on server..."
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -f /tmp/schema.sql && sudo -u postgres psql -d ims -f /tmp/rls.sql"
|
||||
|
||||
echo "==> Cleaning up temp files..."
|
||||
ssh -p "$PORT" "$SERVER" "rm /tmp/schema.sql /tmp/rls.sql"
|
||||
|
||||
echo "==> Schema loaded. Verifying table count..."
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS table_count FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE';\""
|
||||
# Expected: 14 tables (13 original + password_reset_tokens)
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/02-dump-and-restore.sh
|
||||
set -e
|
||||
|
||||
if [ -z "$SUPABASE_DB_URL" ]; then
|
||||
echo "ERROR: SUPABASE_DB_URL not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVER="setia@ims.setia.com.my"
|
||||
PORT=9321
|
||||
DUMP_FILE="/tmp/ims-supabase-dump-$(date +%Y%m%d-%H%M%S).sql"
|
||||
|
||||
echo "==> Dumping Supabase public schema (data only)..."
|
||||
pg_dump \
|
||||
--data-only \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--schema=public \
|
||||
--exclude-table=schema_migrations \
|
||||
"$SUPABASE_DB_URL" \
|
||||
-f "$DUMP_FILE"
|
||||
|
||||
echo "==> Dump written to $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))"
|
||||
|
||||
echo "==> Copying dump to server..."
|
||||
scp -P "$PORT" "$DUMP_FILE" "$SERVER:/tmp/ims-data.sql"
|
||||
|
||||
echo "==> Restoring on server..."
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -f /tmp/ims-data.sql 2>&1 | tail -20"
|
||||
|
||||
echo "==> Cleaning up..."
|
||||
ssh -p "$PORT" "$SERVER" "rm /tmp/ims-data.sql"
|
||||
rm "$DUMP_FILE"
|
||||
|
||||
echo "==> Done. Run db/05-verify.sh to check row counts."
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/bash
|
||||
# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/03-migrate-passwords.sh
|
||||
set -e
|
||||
|
||||
if [ -z "$SUPABASE_DB_URL" ]; then
|
||||
echo "ERROR: SUPABASE_DB_URL not set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVER="setia@ims.setia.com.my"
|
||||
PORT=9321
|
||||
AUTH_DUMP="/tmp/ims-auth-users-$(date +%Y%m%d-%H%M%S).csv"
|
||||
|
||||
echo "==> Exporting auth.users (id + encrypted_password + created_at) from Supabase..."
|
||||
psql "$SUPABASE_DB_URL" -c "\COPY (SELECT id, encrypted_password, created_at FROM auth.users WHERE encrypted_password IS NOT NULL) TO STDOUT WITH CSV HEADER" > "$AUTH_DUMP"
|
||||
|
||||
echo "==> Exported $(wc -l < "$AUTH_DUMP") rows (including header)"
|
||||
|
||||
echo "==> Copying to server..."
|
||||
scp -P "$PORT" "$AUTH_DUMP" "$SERVER:/tmp/auth-users.csv"
|
||||
|
||||
echo "==> Applying password hashes to users table on server..."
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims" <<'ENDSQL'
|
||||
-- Create temp table for the auth data
|
||||
CREATE TEMP TABLE auth_users_import (
|
||||
id UUID,
|
||||
password_hash TEXT,
|
||||
created_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
\COPY auth_users_import FROM '/tmp/auth-users.csv' WITH CSV HEADER;
|
||||
|
||||
-- Update users table
|
||||
UPDATE users u
|
||||
SET
|
||||
password_hash = a.password_hash,
|
||||
email_verified_at = COALESCE(u.email_verified_at, a.created_at)
|
||||
FROM auth_users_import a
|
||||
WHERE u.id = a.id;
|
||||
|
||||
SELECT count(*) AS updated_users FROM users WHERE password_hash IS NOT NULL;
|
||||
ENDSQL
|
||||
|
||||
echo "==> Verifying no NULL password_hash remains..."
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS null_password_users FROM users WHERE password_hash IS NULL;\""
|
||||
# Should be 0
|
||||
|
||||
echo "==> Cleaning up..."
|
||||
ssh -p "$PORT" "$SERVER" "rm /tmp/auth-users.csv"
|
||||
rm "$AUTH_DUMP"
|
||||
|
||||
echo "==> Password migration complete."
|
||||
Executable
+77
@@ -0,0 +1,77 @@
|
||||
#!/bin/bash
|
||||
# Usage:
|
||||
# SUPABASE_URL="https://xxx.supabase.co" \
|
||||
# SUPABASE_SERVICE_ROLE_KEY="eyJ..." \
|
||||
# bash db/04-migrate-evidence.sh
|
||||
set -e
|
||||
|
||||
if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_SERVICE_ROLE_KEY" ]; then
|
||||
echo "ERROR: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY must be set"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SERVER="setia@ims.setia.com.my"
|
||||
PORT=9321
|
||||
REMOTE_DIR="/var/lib/ims/evidence"
|
||||
LOCAL_TMP="/tmp/ims-evidence-migration"
|
||||
|
||||
mkdir -p "$LOCAL_TMP"
|
||||
|
||||
echo "==> Listing all objects in Supabase Storage bucket 'evidence'..."
|
||||
# Paginate: offset 0, limit 1000 (repeat if >1000 files)
|
||||
OBJECTS=$(curl -s \
|
||||
-H "Authorization: Bearer $SUPABASE_SERVICE_ROLE_KEY" \
|
||||
-H "Content-Type: application/json" \
|
||||
"$SUPABASE_URL/storage/v1/object/list/evidence" \
|
||||
-d '{"limit": 1000, "offset": 0, "prefix": ""}')
|
||||
|
||||
COUNT=$(echo "$OBJECTS" | python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo 0)
|
||||
echo "==> Found $COUNT objects"
|
||||
|
||||
if [ "$COUNT" -eq 0 ]; then
|
||||
echo "==> No evidence files to migrate. Exiting."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> Downloading evidence files..."
|
||||
# Export vars so the Python subprocess can access them
|
||||
export LOCAL_TMP SERVER PORT REMOTE_DIR
|
||||
echo "$OBJECTS" | python3 - <<'PYEOF'
|
||||
import sys, json, os, subprocess, urllib.request
|
||||
|
||||
objects = json.load(sys.stdin)
|
||||
supabase_url = os.environ['SUPABASE_URL']
|
||||
service_role_key = os.environ['SUPABASE_SERVICE_ROLE_KEY']
|
||||
local_tmp = os.environ.get('LOCAL_TMP', '/tmp/ims-evidence-migration')
|
||||
server = os.environ['SERVER']
|
||||
port = os.environ['PORT']
|
||||
remote_dir = os.environ['REMOTE_DIR']
|
||||
|
||||
for i, obj in enumerate(objects):
|
||||
name = obj['name'] # e.g. "userId/incidentId/stage/filename.jpg"
|
||||
local_path = os.path.join(local_tmp, name)
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
|
||||
# Download from Supabase Storage
|
||||
url = f"{supabase_url}/storage/v1/object/evidence/{name}"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {service_role_key}"})
|
||||
with urllib.request.urlopen(req) as resp, open(local_path, 'wb') as f:
|
||||
f.write(resp.read())
|
||||
|
||||
print(f"[{i+1}/{len(objects)}] Downloaded: {name}")
|
||||
PYEOF
|
||||
|
||||
echo "==> Uploading evidence files to server..."
|
||||
# Create remote directory structure
|
||||
ssh -p "$PORT" "$SERVER" "mkdir -p $REMOTE_DIR"
|
||||
|
||||
# rsync preserves directory structure
|
||||
rsync -az --progress -e "ssh -p $PORT" "$LOCAL_TMP/" "$SERVER:$REMOTE_DIR/"
|
||||
|
||||
echo "==> Cleaning up local temp..."
|
||||
rm -rf "$LOCAL_TMP"
|
||||
|
||||
echo "==> Verifying file count on server..."
|
||||
ssh -p "$PORT" "$SERVER" "find $REMOTE_DIR -type f | wc -l"
|
||||
|
||||
echo "==> Evidence migration complete."
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/bash
|
||||
# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/05-verify.sh
|
||||
set -e
|
||||
|
||||
SERVER="setia@ims.setia.com.my"
|
||||
PORT=9321
|
||||
|
||||
echo "=== IMS Data Migration Verification ==="
|
||||
echo ""
|
||||
|
||||
echo "--- Table row counts on NEW server ---"
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims" <<'ENDSQL'
|
||||
SELECT
|
||||
tablename,
|
||||
n_live_tup AS row_count
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY tablename;
|
||||
ENDSQL
|
||||
|
||||
if [ -n "$SUPABASE_DB_URL" ]; then
|
||||
echo ""
|
||||
echo "--- Table row counts on SUPABASE (source) ---"
|
||||
psql "$SUPABASE_DB_URL" <<'ENDSQL'
|
||||
SELECT
|
||||
tablename,
|
||||
n_live_tup AS row_count
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY tablename;
|
||||
ENDSQL
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Users with NULL password_hash (should be 0) ---"
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) FROM users WHERE password_hash IS NULL;\""
|
||||
|
||||
echo ""
|
||||
echo "--- Evidence files row count vs filesystem count ---"
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS db_evidence_rows FROM evidence_files WHERE deleted = false;\""
|
||||
ssh -p "$PORT" "$SERVER" "find /var/lib/ims/evidence -type f | wc -l"
|
||||
|
||||
echo ""
|
||||
echo "--- pgvector embeddings check ---"
|
||||
ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS incidents_with_embedding FROM incidents WHERE embedding IS NOT NULL;\""
|
||||
|
||||
echo ""
|
||||
echo "=== Verification complete. Compare counts above. ==="
|
||||
@@ -478,6 +478,21 @@ VALUES
|
||||
('00000000-0000-0000-0000-000000000012'::UUID, '00000000-0000-0000-0000-000000000001'::UUID, 'Loading Bay', 'scw1-loading-bay-qr-2026')
|
||||
ON CONFLICT (id) DO NOTHING;
|
||||
|
||||
-- ---------------------------------------------------------------------------
|
||||
-- Password reset tokens (Phase 3 — custom auth)
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
used_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS password_reset_tokens_user_id_idx ON password_reset_tokens(user_id);
|
||||
|
||||
-- Seed: app_settings (final state after all migrations)
|
||||
INSERT INTO app_settings (key, value) VALUES
|
||||
('DEEPSEEK_API_KEY', ''),
|
||||
|
||||
Reference in New Issue
Block a user