Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 17afcac

Browse files
Rajandeep98Rajandeep98
authored
34854 - Load Officers Data Migration (#4768)
1 parent 26cfc7a commit 17afcac

2 files changed

Lines changed: 241 additions & 1 deletion

File tree

‎data-tool/notebooks/backfill_contact_email_auth/.env.sample‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,4 +22,5 @@ ACCOUNT_SVC_CLIENT_SECRET=
2222
ACCOUNT_SVC_TIMEOUT=
2323

2424
MIG_BATCH_ID=
25-
ENVIRONMENTS=
25+
ENVIRONMENTS=
26+
TARGET_SCHEMA=

‎data-tool/notebooks/backfill_contact_email_auth/load_officers.ipynb‎

Lines changed: 239 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,239 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "0c05ca4d",
6+
"metadata": {},
7+
"source": [
8+
"# BACKFILL Officers Data\n",
9+
"\n",
10+
"## Overview\n",
11+
"Add Officers Data\n",
12+
"- Get id list per batch and group from corp processing entries\n",
13+
"- Create a range for making Call to Function to load officers under parties"
14+
]
15+
},
16+
{
17+
"cell_type": "code",
18+
"execution_count": null,
19+
"id": "495c1fa6",
20+
"metadata": {},
21+
"outputs": [],
22+
"source": [
23+
"%pip install pandas requests\n",
24+
"%pip install sqlalchemy>=2.0\n",
25+
"%pip install psycopg2-binary\n",
26+
"%pip install python-dotenv"
27+
]
28+
},
29+
{
30+
"cell_type": "markdown",
31+
"id": "2e7f8781",
32+
"metadata": {},
33+
"source": [
34+
"# Load Configurations"
35+
]
36+
},
37+
{
38+
"cell_type": "code",
39+
"execution_count": null,
40+
"id": "3f0836c9",
41+
"metadata": {},
42+
"outputs": [],
43+
"source": [
44+
"import os\n",
45+
"from datetime import datetime\n",
46+
"from typing import Optional\n",
47+
"\n",
48+
"import pandas as pd\n",
49+
"from sqlalchemy import create_engine, text\n",
50+
"from sqlalchemy.exc import SQLAlchemyError, OperationalError\n",
51+
"from sqlalchemy.engine import Engine\n",
52+
"from dotenv import load_dotenv\n",
53+
"\n",
54+
"# Load environment variables\n",
55+
"load_dotenv()\n",
56+
"print(\"Environment variables loaded successfully.\")"
57+
]
58+
},
59+
{
60+
"cell_type": "markdown",
61+
"id": "7b8aeaad",
62+
"metadata": {},
63+
"source": [
64+
"## Database Configuration\n",
65+
"\n",
66+
"Configure connections to:\n",
67+
"- **colin_extract**: Target database for `corp_processing` table"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": null,
73+
"id": "5b5a0e55",
74+
"metadata": {},
75+
"outputs": [],
76+
"source": [
77+
"DATABASE_CONFIG = {\n",
78+
" 'business': {\n",
79+
" 'username': os.getenv(\"DATABASE_USERNAME\"),\n",
80+
" 'password': os.getenv(\"DATABASE_PASSWORD\"),\n",
81+
" 'host': os.getenv(\"DATABASE_HOST\"),\n",
82+
" 'port': os.getenv(\"DATABASE_PORT\"),\n",
83+
" 'name': os.getenv(\"DATABASE_NAME\")\n",
84+
" }\n",
85+
"}\n",
86+
"\n",
87+
"# Build connection URIs\n",
88+
"for db_key, db_config in DATABASE_CONFIG.items():\n",
89+
" # Validate config\n",
90+
" missing_keys = [k for k, v in db_config.items() if v is None]\n",
91+
" if missing_keys:\n",
92+
" print(f\"{db_key.upper()}: Missing environment variables for: {missing_keys}\")\n",
93+
"\n",
94+
" # Build PostgreSQL URI\n",
95+
" uri = f\"postgresql://{db_config['username']}:{db_config['password']}@{db_config['host']}:{db_config['port']}/{db_config['name']}\"\n",
96+
" DATABASE_CONFIG[db_key] = {'uri': uri}\n",
97+
"\n",
98+
" print(\"Database configurations built successfully.\")\n",
99+
"\n",
100+
"TARGET_SCHEMA = os.getenv(\"TARGET_SCHEMA\")\n",
101+
"MIG_BATCH_ID = os.getenv(\"MIG_BATCH_ID\")\n",
102+
"ENVIRONMENTS = os.getenv(\"ENVIRONMENTS\")\n",
103+
"print(\"Service URLs and credentials loaded successfully.\")"
104+
]
105+
},
106+
{
107+
"cell_type": "markdown",
108+
"id": "546188b6",
109+
"metadata": {},
110+
"source": [
111+
"## Get Identifier for Batch and Group"
112+
]
113+
},
114+
{
115+
"cell_type": "code",
116+
"execution_count": null,
117+
"id": "01373559",
118+
"metadata": {},
119+
"outputs": [],
120+
"source": [
121+
"engines = {}\n",
122+
"\n",
123+
"for db_key, config in DATABASE_CONFIG.items():\n",
124+
" try:\n",
125+
" print(f\"Creating engine for {db_key.upper()}...\")\n",
126+
" engine = create_engine(config['uri'])\n",
127+
"\n",
128+
" # Test connection\n",
129+
" with engine.connect() as conn:\n",
130+
" conn.execute(text(\"SELECT 1\"))\n",
131+
"\n",
132+
" engines[db_key] = engine\n",
133+
" print(f\"✓ {db_key.upper()} database engine created and tested successfully.\")\n",
134+
"\n",
135+
" except OperationalError as e:\n",
136+
" print(f\"✗ {db_key.upper()} database connection failed: {e}\")\n",
137+
" raise\n",
138+
" except SQLAlchemyError as e:\n",
139+
" print(f\"✗ {db_key.upper()} database engine creation failed: {e}\")\n",
140+
" raise\n",
141+
" except Exception as e:\n",
142+
" print(f\"✗ {db_key.upper()} unexpected error: {e}\")\n",
143+
" raise\n",
144+
"\n",
145+
"print(\"=\"*50)\n",
146+
"print(\"All database engines ready for use.\")\n",
147+
"print(\"=\"*50)"
148+
]
149+
},
150+
{
151+
"cell_type": "code",
152+
"execution_count": null,
153+
"id": "40a41d56",
154+
"metadata": {},
155+
"outputs": [],
156+
"source": [
157+
"IDENTIFIERS_RANGE_QUERY = \"\"\"\n",
158+
"SELECT id, corp_num\n",
159+
"FROM colin_extract.corp_processing cp\n",
160+
"WHERE processed_status = 'COMPLETED'\n",
161+
"AND mig_batch_id = :mig_batch_id\n",
162+
"AND environment = :environment\n",
163+
"-- LIMIT 1\n",
164+
"\"\"\"\n",
165+
"\n",
166+
"def query_identifiers(engine: Engine, mig_batch_id: int, environment: str) -> pd.DataFrame:\n",
167+
" try:\n",
168+
" with engine.connect() as conn:\n",
169+
" result = conn.execute(text(IDENTIFIERS_RANGE_QUERY), {\"mig_batch_id\": mig_batch_id, \"environment\": environment})\n",
170+
" identifiers_df = pd.DataFrame(result.fetchall(), columns=result.keys())\n",
171+
" print(f\"✓ Successfully queried identifiers. Total records: {len(identifiers_df)}\")\n",
172+
" return identifiers_df\n",
173+
" except SQLAlchemyError as e:\n",
174+
" print(f\"✗ Error querying identifiers: {e}\")\n",
175+
" raise\n",
176+
" except Exception as e:\n",
177+
" print(f\"✗ Unexpected error querying identifiers: {e}\")\n",
178+
" raise\n",
179+
"\n",
180+
"identifier = query_identifiers(engines['business'], MIG_BATCH_ID, ENVIRONMENTS)\n",
181+
"print(f\"Total identifiers retrieved: {len(identifier)}\")"
182+
]
183+
},
184+
{
185+
"cell_type": "code",
186+
"execution_count": null,
187+
"id": "a6930d4c",
188+
"metadata": {},
189+
"outputs": [],
190+
"source": [
191+
"OFFICERS_RANGE_FUNCTION = \"\"\"\n",
192+
"SELECT public.colin_tombstone_officers_range(:environment, :first_id, :last_id);\n",
193+
"\"\"\"\n",
194+
"first_id = identifier['id'].iloc[0].item()\n",
195+
"last_id = identifier['id'].iloc[-1].item()\n",
196+
"print(f\"Loading Officers from ID {first_id} TO {last_id}\")\n",
197+
"def update_officers(engine: Engine, first_id: int, last_id: int, environment: str) -> pd.DataFrame:\n",
198+
" try:\n",
199+
" with engine.connect() as conn:\n",
200+
" result = conn.execute(text(OFFICERS_RANGE_FUNCTION), { \"environment\": environment, \"first_id\": first_id, \"last_id\": last_id})\n",
201+
" conn.commit()\n",
202+
" value = result.scalar()\n",
203+
" print(f\"Officers Result: {value}\")\n",
204+
" return value\n",
205+
" except SQLAlchemyError as e:\n",
206+
" print(f\"✗ Error querying officers data: {e}\")\n",
207+
" raise\n",
208+
" except Exception as e:\n",
209+
" print(f\"✗ Unexpected error querying identifiers: {e}\")\n",
210+
" raise\n",
211+
"\n",
212+
"officers_load = update_officers(engines['business'], first_id, last_id, ENVIRONMENTS)\n",
213+
"print(f\"Total updated officers: {officers_load}\")\n",
214+
"\n"
215+
]
216+
}
217+
],
218+
"metadata": {
219+
"kernelspec": {
220+
"display_name": "Python 3",
221+
"language": "python",
222+
"name": "python3"
223+
},
224+
"language_info": {
225+
"codemirror_mode": {
226+
"name": "ipython",
227+
"version": 3
228+
},
229+
"file_extension": ".py",
230+
"mimetype": "text/x-python",
231+
"name": "python",
232+
"nbconvert_exporter": "python",
233+
"pygments_lexer": "ipython3",
234+
"version": "3.9.6"
235+
}
236+
},
237+
"nbformat": 4,
238+
"nbformat_minor": 5
239+
}

0 commit comments

Comments
 (0)