# City of Boston — ArcGIS (no key)
# ============================================

# FY26 — Single parcel
curl -s -G "https://gis.bostonplans.org/hosting/rest/services/FY26_Parcels_Detailed/FeatureServer/0/query" \
  --data-urlencode "where=PID='<PID>'" \
  --data-urlencode "outFields=*" \
  --data-urlencode "returnGeometry=false" \
  --data-urlencode "f=json"

# FY26 — All parcels in ZIP
curl -s -G "https://gis.bostonplans.org/hosting/rest/services/FY26_Parcels_Detailed/FeatureServer/0/query" \
  --data-urlencode "where=ZIP_CODE='02134'" \
  --data-urlencode "outFields=PID,FULL_ADDRESS,LU,LU_DESC,LIVING_AREA,TOTAL_VALUE,YR_BUILT,OWNER,OWN_OCC,ZIP_CODE" \
  --data-urlencode "orderByFields=TOTAL_VALUE DESC" \
  --data-urlencode "returnGeometry=false" \
  --data-urlencode "f=json"

# FY26 — All parcels on a street
curl -s -G "https://gis.bostonplans.org/hosting/rest/services/FY26_Parcels_Detailed/FeatureServer/0/query" \
  --data-urlencode "where=ST_NAME='<STREET>'" \
  --data-urlencode "outFields=*" \
  --data-urlencode "orderByFields=TOTAL_VALUE DESC" \
  --data-urlencode "returnGeometry=false" \
  --data-urlencode "f=json"

# FY26 — Filter by land use
# where=LU IN ('R2','R3') AND ZIP_CODE='02134'

# FY25 — Different endpoint, uses GIS_ID not PID
curl -s -G "https://gisportal.boston.gov/arcgis/rest/services/Assessing/Property_Assessment_FY25/FeatureServer/0/query" \
  --data-urlencode "where=GIS_ID='<PID>'" \
  --data-urlencode "outFields=*" \
  --data-urlencode "returnGeometry=false" \
  --data-urlencode "f=json"

# Historical year discovery:
#   https://www.arcgis.com/sharing/rest/search
#   Search: "Assessing" or "Property_Assessment"


# City of Boston — CKAN (no key)
# ============================================

# Building permits by street name
curl -s -G "https://data.boston.gov/api/3/action/datastore_search" \
  --data-urlencode "resource_id=6ddcd912-32a0-43df-9908-63574f8c7e77" \
  --data-urlencode "q=<STREET_NAME>" \
  --data-urlencode "limit=100"
# Filter results to specific address after fetching.

# Annual assessment CSVs (FY2013–FY2026):
#   https://data.boston.gov/dataset/e02c44d2-3c64-459c-8fe2-e1ce5f38a035
#   Filter by PID. Older CSVs may store tax as cents-integer (÷100).


# Zillow — Apify (needs $APIFY_API_KEY)
# ============================================

# General 3-step pattern for all Apify actors:

# 1. Start
curl -s -X POST "https://api.apify.com/v2/acts/<ACTOR>/runs?token=$APIFY_API_KEY" \
  -H "Content-Type: application/json" \
  -d '<CONFIG>'
# → .data.id = RUN_ID, .data.defaultDatasetId = DATASET_ID

# 2. Poll
curl -s "https://api.apify.com/v2/acts/<ACTOR>/runs/$RUN_ID?token=$APIFY_API_KEY"
# → .data.status, poll every 10s until "SUCCEEDED"

# 3. Fetch
curl -s "https://api.apify.com/v2/acts/<ACTOR>/runs/$RUN_ID/dataset/items?token=$APIFY_API_KEY&format=json"

# --- Detail Scraper: maxcopell/zillow-detail-scraper ---
# Start config:
# {
#   "propertyStatus": "RECENTLY_SOLD",
#   "startUrls": [{"url": "https://www.zillow.com/homedetails/..."}]
# }
# Result is array. Single URL → single-element array.

# --- ZIP Search: maxcopell/zillow-zip-search ---
# Start config: {"zipCodes":["02134"]}
# Returns basic property list (no price history). Use to cross-ref addresses → ZPIDs.

# --- Google Search Scraper: apify/google-search-scraper ---
# Query: "<FULL_ADDRESS> zillow"
# Extracts Zillow URL + ZPID from organic results.
# Proven: "30 Seattle St Allston MA 02134 zillow" → ZPID 59086989.

# --- Batch: pass multiple URLs to Detail Scraper in one run ---
# "startUrls": [{"url":"..."}, {"url":"..."}, ...]
# Split results by address afterward.


# MA Secretary of State — Puppeteer (Incapsula, headful Chrome)
# ==============================================================

# url: https://corp.sec.state.ma.us/CorpWeb/CorpSearch/CorpSearch.aspx
# blocked to curl → launch headful Chrome (headless: false)

# flow:
#   1. goto CorpSearch.aspx, wait networkidle2
#   2. fill input[name*="EntityName"] with name, fallback: first visible input[text]
#   3. click input[value="Search Corporations"], wait navigation
#   4. find <a> whose textContent includes entity name (upper-cased), click, wait nav
#      fallback: first <a> in first table row
#   5. extract from CorpSummary:
#        querySelectorAll('table tr, .formTable tr, .corpTable tr')
#        for each row with ≥2 cells: label = cells[0].text (strip ':'), value = cells[1].text
#        also grab body.innerText(5000) as rawText
#   6. save {fields[], rawText, url}


# Suffolk Registry of Deeds — Puppeteer (ASP.NET __doPostBack)
# =============================================================

# url: https://www.masslandrecords.com/suffolk/
# CRITICAL: ASP.NET WebForms. Changing <select> triggers server __doPostBack.
# Must use page.select() on native element. Setting .value does NOT work.
# After each page.select(), wait 2s for postback.

# flow:
#   1. goto masslandrecords.com/suffolk, wait networkidle2 + 3s
#   2. page.select('#SearchCriteriaOffice1_DDL_OfficeName', 'Recorded Land'), wait 2s
#   3. page.select('#SearchCriteriaName1_DDL_SearchName', '<MODE>'), wait 2s
#   4. fill form fields (see below), click #SearchFormEx1_btnSearch, wait nav
#   5. extract: querySelectorAll('table') → rows → cells → textContent
#      filter tables containing: deed, grant, sale, book, page, date, price, consideration
#      capture links (exclude javascript:), body.innerText(8000), window.location.href

# --- Name Search mode ---
# searchType: 'Recorded Land Name Search'
# last:  click + type '#SearchFormEx1_ACSTextBox_LastName1'
# first: click + type '#SearchFormEx1_ACSTextBox_FirstName1'

# --- Property Search mode ---
# searchType: 'Recorded Land Property Search'
# number: click + type '#SearchFormEx1_ACSTextBox_StreetNumber'
# name:   click + type '#SearchFormEx1_ACSTextBox_StreetName'

# --- Deed click-through from results ---
# find <a> matching book/page text (e.g. '18656/282'), click, wait 3s
# extract: grantor, grantee, consideration, file date, type, book/page


# Consolidation — assembly pseudo-code
# ============================================

# inputs:
#   Scrape/zillow/42-44-hano.json                     (array, take [0])
#   Scrape/city-of-boston/42-44-hano-assessment.json
#   Scrape/city-of-boston/42-44-hano-tax-history.json
#   Scrape/city-of-boston/42-44-hano-permits.json
#   Scrape/secretary-of-state/hano-dev-holdings-llc.txt   (parse rawText)

# 1. zillow: extract zpid, address, homeType, bedrooms, bathrooms, livingArea,
#    lotSize, yearBuilt, zestimate, dateSold, lastSoldPrice, daysOnZillow,
#    description, lat, lon, priceHistory[], schools[], resoFacts{}

# 2. assessment: extract fullAddress, owner, mailingAddress, landSqft, grossAreaSqft,
#    livingAreaSqft, yearBuilt, yearRemodeled, stories, bedrooms, fullBaths, halfBaths,
#    kitchens, totalRooms, heating, AC, roof, exterior, interiorCondition,
#    exteriorCondition, overallCondition, assessment.fy2026{landValue,buildingValue,
#    totalValue,grossTax}

# 3. taxHistory: extract history[] → owner timeline FY2013→FY2026

# 4. permits: extract permits[], compute summary{total, totalValue, open, closed}

# 5. sos: parse fields[] and rawText for ID number, organization date, dissolution date,
#    registered agent (name+addr), managers[], real property contacts[], records address

# 6. discrepancies: lotSize(city vs zillow), saleRecord(city FY2019-20 vs zillow 2018-05)

# 7. assemble profile: {meta, property, valuation, ownership, contactInformation,
#    buildingPermits, permitsSummary, zillowDetails, discrepancies}

# 8. write: Consolidate/44-hano-profile.json (JSON.stringify, indent=2)
#           Consolidate/44-hano-profile.txt (formatted tables, separators)


# Neighbor pipeline
# ============================================

# for each R2 PID on Hano St (from street query, filter LU='R2'):
#   1. arcgis per-PID query → neighbors/<PID>-assessment.json
#   2. CKAN permit search by street name, filter to address → neighbors/<PID>-permits.json
#   3. tax history from annual CSV, filter by PID → neighbors/<PID>-tax-history.json
#   4. google-search-scraper: "<FULL_ADDRESS> zillow" → ZPID
#      zillow-detail-scraper batch: all ZPIDs in one run → neighbors/<ADDRESS>.json
#   5. filter priceHistory to arm's-length sales (no trusts, no LLC-LLC, no $0, no quitclaim)
#   6. assemble per-neighbor profile matching subject schema