need version id

datamade · Mar 4, 2024 · f2b37cc · f2b37cc
1 parent b484a22
commit f2b37cc
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -92,10 +92,10 @@ data/intermediate/lobbyists.csv : data/raw/lobbyists.csv
 	USING (MemberID, MemberVersionID, Year, ClientID)" < $< > $@
 
 data/intermediate/clients.csv : data/raw/clients.csv
-	csvsql --query "SELECT ClientID, MAX(ClientName) AS ClientName FROM STDIN GROUP BY ClientID" < $< > $@
+	csvsql --query "SELECT ClientID, ClientVersionID, MAX(ClientName) AS ClientName FROM STDIN GROUP BY ClientID" < $< > $@
 
 data/intermediate/filings.csv : data/raw/lobbyists.csv
-	csvsql --query "SELECT DISTINCT MemberID FROM STDIN" < $< | \
+	csvsql --query "SELECT DISTINCT MemberID, MemberVersionID FROM STDIN" < $< | \
 	python -m scrapers.lobbyist.scrape_filings > $@
 
 data/raw/lobbyists.csv : data/intermediate/clients.csv

diff --git a/scrapers/lobbyist/scrape_filings.py b/scrapers/lobbyist/scrape_filings.py
@@ -64,6 +64,7 @@
 for row in tqdm(reader):
     _payload = payload.copy()
     _payload["LobbyistID"] = row["MemberID"]
+    _payload["LobbyistVersionID"] = row["MemberVersionID"]
 
     response = s.post(
         "https://login.cfis.sos.state.nm.us/api//ExploreClients/Fillings",

diff --git a/scrapers/lobbyist/scrape_lobbyists.py b/scrapers/lobbyist/scrape_lobbyists.py
@@ -76,6 +76,7 @@
 for row in tqdm(reader):
     _payload = payload.copy()
     _payload["ClientID"] = row["ClientID"]
+    _payload["ClientVersionID"] = str(int(float(row["ClientVersionID"])))
 
     response = s.post(
         "https://login.cfis.sos.state.nm.us/api//ClientDetails/LobbyistsByClientList",