""" FLHIP Scraper v3 - Configuration Using Claude API for intelligent lead extraction Full RSS feed list from all sources """ import os from pathlib import Path from dotenv import load_dotenv load_dotenv(Path(__file__).parent.parent / '.env') from datetime import datetime # ============================================================================= # API KEYS (set via environment variables) # ============================================================================= ANTHROPIC_API_KEY = os.getenv("ANTHROPIC_API_KEY", "") # ============================================================================= # DATABASE CONFIG # ============================================================================= DB_CONFIG = { "host": os.getenv("DB_HOST", "localhost"), "user": os.getenv("DB_USER", "root"), "password": os.getenv("DB_PASSWORD", ""), "database": os.getenv("DB_NAME", "restaurant_openings"), "charset": "utf8mb4" } # ============================================================================= # EMAIL CONFIG # ============================================================================= EMAIL_CONFIG = { "smtp_host": os.getenv("SMTP_HOST", "smtp.gmail.com"), "smtp_port": int(os.getenv("SMTP_PORT", "587")), "smtp_user": os.getenv("SMTP_USER", ""), "smtp_password": os.getenv("SMTP_PASSWORD", ""), "from_address": os.getenv("EMAIL_FROM", ""), "to_addresses": os.getenv("EMAIL_TO", "").split(","), } # ============================================================================= # CLAUDE API CONFIG # ============================================================================= CLAUDE_MODEL = "claude-sonnet-4-20250514" CLAUDE_MAX_TOKENS = 2000 CLAUDE_TEMPERATURE = 0 # ============================================================================= # WHATNOW FEEDS - High quality source (32 cities) # ============================================================================= WHATNOW_FEEDS = [ {"name": "WhatNow Atlanta", "url": "https://whatnow.com/atlanta/feed/", "type": "whatnow"}, {"name": "WhatNow Austin", "url": "https://whatnow.com/austin/feed/", "type": "whatnow"}, {"name": "WhatNow Boston", "url": "https://whatnow.com/boston/feed/", "type": "whatnow"}, {"name": "WhatNow Charlotte", "url": "https://whatnow.com/charlotte/feed/", "type": "whatnow"}, {"name": "WhatNow Chicago", "url": "https://whatnow.com/chicago/feed/", "type": "whatnow"}, {"name": "WhatNow Columbus", "url": "https://whatnow.com/columbus/feed/", "type": "whatnow"}, {"name": "WhatNow Dallas", "url": "https://whatnow.com/dallas/feed/", "type": "whatnow"}, {"name": "WhatNow Denver", "url": "https://whatnow.com/denver/feed/", "type": "whatnow"}, {"name": "WhatNow Detroit", "url": "https://whatnow.com/detroit/feed/", "type": "whatnow"}, {"name": "WhatNow Houston", "url": "https://whatnow.com/houston/feed/", "type": "whatnow"}, {"name": "WhatNow Jacksonville", "url": "https://whatnow.com/jacksonville/feed/", "type": "whatnow"}, {"name": "WhatNow Las Vegas", "url": "https://whatnow.com/las-vegas/feed/", "type": "whatnow"}, {"name": "WhatNow Los Angeles", "url": "https://whatnow.com/los-angeles/feed/", "type": "whatnow"}, {"name": "WhatNow Memphis", "url": "https://whatnow.com/memphis/feed/", "type": "whatnow"}, {"name": "WhatNow Miami", "url": "https://whatnow.com/miami/feed/", "type": "whatnow"}, {"name": "WhatNow Minneapolis", "url": "https://whatnow.com/minneapolis/feed/", "type": "whatnow"}, {"name": "WhatNow Nashville", "url": "https://whatnow.com/nashville/feed/", "type": "whatnow"}, {"name": "WhatNow New Orleans", "url": "https://whatnow.com/new-orleans/feed/", "type": "whatnow"}, {"name": "WhatNow New York", "url": "https://whatnow.com/new-york/feed/", "type": "whatnow"}, {"name": "WhatNow Orlando", "url": "https://whatnow.com/orlando/feed/", "type": "whatnow"}, {"name": "WhatNow Philadelphia", "url": "https://whatnow.com/philadelphia/feed/", "type": "whatnow"}, {"name": "WhatNow Phoenix", "url": "https://whatnow.com/phoenix/feed/", "type": "whatnow"}, {"name": "WhatNow Portland", "url": "https://whatnow.com/portland/feed/", "type": "whatnow"}, {"name": "WhatNow Raleigh", "url": "https://whatnow.com/raleigh/feed/", "type": "whatnow"}, {"name": "WhatNow Sacramento", "url": "https://whatnow.com/sacramento/feed/", "type": "whatnow"}, {"name": "WhatNow San Antonio", "url": "https://whatnow.com/san-antonio/feed/", "type": "whatnow"}, {"name": "WhatNow San Diego", "url": "https://whatnow.com/san-diego/feed/", "type": "whatnow"}, {"name": "WhatNow San Francisco", "url": "https://whatnow.com/san-francisco/feed/", "type": "whatnow"}, {"name": "WhatNow Seattle", "url": "https://whatnow.com/seattle/feed/", "type": "whatnow"}, {"name": "WhatNow St Louis", "url": "https://whatnow.com/st-louis/feed/", "type": "whatnow"}, {"name": "WhatNow Tampa", "url": "https://whatnow.com/tampa/feed/", "type": "whatnow"}, {"name": "WhatNow Washington DC", "url": "https://whatnow.com/washington-dc/feed/", "type": "whatnow"}, ] # ============================================================================= # EATER FEEDS - Good restaurant coverage (23 feeds) # ============================================================================= EATER_FEEDS = [ {"name": "Eater National", "url": "https://www.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Atlanta", "url": "https://atlanta.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Austin", "url": "https://austin.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Boston", "url": "https://boston.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Chicago", "url": "https://chicago.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Dallas", "url": "https://dallas.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Denver", "url": "https://denver.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Detroit", "url": "https://detroit.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Houston", "url": "https://houston.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Las Vegas", "url": "https://vegas.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Los Angeles", "url": "https://la.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Miami", "url": "https://miami.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Minneapolis", "url": "https://twincities.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Nashville", "url": "https://nashville.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater New Orleans", "url": "https://nola.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater New York", "url": "https://ny.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Philadelphia", "url": "https://philly.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Phoenix", "url": "https://phoenix.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Portland", "url": "https://pdx.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater San Diego", "url": "https://sandiego.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater San Francisco", "url": "https://sf.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater Seattle", "url": "https://seattle.eater.com/rss/index.xml", "type": "eater"}, {"name": "Eater DC", "url": "https://dc.eater.com/rss/index.xml", "type": "eater"}, ] # ============================================================================= # INDUSTRY FEEDS - Restaurant industry news (7 feeds) # ============================================================================= INDUSTRY_FEEDS = [ {"name": "Nations Restaurant News", "url": "https://www.nrn.com/rss.xml", "type": "industry"}, {"name": "RestaurantNews.com", "url": "https://www.restaurantnews.com/feed/", "type": "industry"}, {"name": "Restaurant Business Online", "url": "https://www.restaurantbusinessonline.com/rss.xml", "type": "industry"}, {"name": "QSR Magazine", "url": "https://www.qsrmagazine.com/rss.xml", "type": "industry"}, {"name": "FSR Magazine", "url": "https://www.fsrmagazine.com/rss.xml", "type": "industry"}, {"name": "Fast Food News EIN", "url": "https://fastfood.einnews.com/rss/latest", "type": "industry"}, {"name": "Restaurants EIN Newswire", "url": "https://restaurants.einnews.com/rss/latest", "type": "industry"}, ] # ============================================================================= # GOOGLE ALERTS FEEDS - City + Keyword specific (93 feeds) # ============================================================================= GOOGLE_ALERTS_FEEDS = [ # Major Cities {"name": "San Francisco restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/9747150454296465331", "type": "google"}, {"name": "Atlanta restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15881591529327843012", "type": "google"}, {"name": "Houston restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6322754630275521756", "type": "google"}, {"name": "Miami restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/12770637845834852795", "type": "google"}, {"name": "Nashville restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15200996147570608561", "type": "google"}, {"name": "Knoxville restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/9403912304759967718", "type": "google"}, {"name": "Philadelphia restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/7368087200454389092", "type": "google"}, {"name": "Pittsburgh restaurants opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/9962240785462582314", "type": "google"}, {"name": "New Orleans restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/11652319189724991647", "type": "google"}, {"name": "Boston restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5648699214771925006", "type": "google"}, {"name": "Las Vegas restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2771843675416648673", "type": "google"}, {"name": "Washington DC restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8313436245565896150", "type": "google"}, {"name": "Phoenix restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/12785288915211581706", "type": "google"}, {"name": "Minneapolis restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/11533645451370743128", "type": "google"}, {"name": "St Louis restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/1336232583057497068", "type": "google"}, {"name": "San Antonio restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5545463520974358596", "type": "google"}, {"name": "Charleston restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/3662225450469670422", "type": "google"}, {"name": "Kansas City restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2319084977322857949", "type": "google"}, {"name": "Little Rock restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2858099941421943288", "type": "google"}, {"name": "Fayetteville Arkansas new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/1305189659041682719", "type": "google"}, {"name": "Bentonville restaurant coming soon", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16728570080742112449", "type": "google"}, {"name": "Northwest Arkansas new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2090596333920091159", "type": "google"}, {"name": "Tulsa new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16658494239145762419", "type": "google"}, {"name": "Oklahoma City restaurant coming soon", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5078923530688755450", "type": "google"}, {"name": "Norman Oklahoma restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6897162181687878548", "type": "google"}, {"name": "Fort Worth restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/17148799242822034117", "type": "google"}, {"name": "Austin New Restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/18440250561498709522", "type": "google"}, {"name": "Austin Restaurants Signed Lease", "url": "https://www.google.com/alerts/feeds/14453255716137723784/14999533757712825969", "type": "google"}, {"name": "Austin Restaurants Coming Soon", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16105551133591885866", "type": "google"}, {"name": "Birmingham restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2025281226710467581", "type": "google"}, {"name": "Memphis restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15764261651767251501", "type": "google"}, {"name": "Savannah restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/185040786323029145", "type": "google"}, {"name": "Indianapolis restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13376834196197467408", "type": "google"}, {"name": "Baltimore restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8764559500501737103", "type": "google"}, {"name": "Sacramento restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/14384663106958368477", "type": "google"}, {"name": "Salt Lake City restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16899527990068056877", "type": "google"}, {"name": "Albuquerque restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/4668565503612778772", "type": "google"}, {"name": "Charlotte restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/12359658562586185107", "type": "google"}, {"name": "Denver restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/86364112781402076", "type": "google"}, {"name": "Los Angeles dining opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/9999865546228957076", "type": "google"}, {"name": "Portland new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15223414411171277560", "type": "google"}, {"name": "Seattle new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/10459463923672684991", "type": "google"}, {"name": "new restaurants in Chicago", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13139921225738312942", "type": "google"}, {"name": "restaurants opening soon Dallas", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13392887392075319064", "type": "google"}, {"name": "new restaurant San Diego", "url": "https://www.google.com/alerts/feeds/01056373355710472933/14893009834191746502", "type": "google"}, {"name": "restaurant opening Austin", "url": "https://www.google.com/alerts/feeds/14453255716137723784/18143979095269508136", "type": "google"}, # State-wide {"name": "South Carolina restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6222282312829287660", "type": "google"}, {"name": "Tennessee restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13378578706162741637", "type": "google"}, {"name": "Texas restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15379574765426002240", "type": "google"}, {"name": "Florida restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6470895804223081220", "type": "google"}, {"name": "New York restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/11512372514533613917", "type": "google"}, {"name": "Ohio restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5109551343554023239", "type": "google"}, {"name": "Oklahoma restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5021000032483330791", "type": "google"}, {"name": "California restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/10494176384702392870", "type": "google"}, {"name": "Colorado restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/17752302210320802060", "type": "google"}, {"name": "Michigan restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/18004289768553444930", "type": "google"}, {"name": "Montana restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/864781464920452263", "type": "google"}, {"name": "Arizona restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/17197259625698739969", "type": "google"}, # Keyword-based {"name": "new vegan restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2556135190463454981", "type": "google"}, {"name": "restaurant grand opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8309535707417469840", "type": "google"}, {"name": "fast food expansion", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13633441662581949843", "type": "google"}, {"name": "fine dining expansion", "url": "https://www.google.com/alerts/feeds/14453255716137723784/11587145333654236677", "type": "google"}, {"name": "restaurant chain growth", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6467645705275380693", "type": "google"}, {"name": "anticipated restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/17362621806676041158", "type": "google"}, {"name": "New staff needed restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6056611834077371865", "type": "google"}, {"name": "new restaurant florida", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5988692147178041638", "type": "google"}, {"name": "new restaurant texas", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5224970958851647533", "type": "google"}, {"name": "new restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/7223986474079465593", "type": "google"}, {"name": "new restaurant new york", "url": "https://www.google.com/alerts/feeds/14453255716137723784/3088834190865104067", "type": "google"}, {"name": "new restaurant tennessee", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13931695517873812989", "type": "google"}, {"name": "restaurant buildout", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6629753073005352897", "type": "google"}, {"name": "restaurant coming", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16955099908585619718", "type": "google"}, {"name": "restaurant construction", "url": "https://www.google.com/alerts/feeds/14453255716137723784/6329052997136349253", "type": "google"}, {"name": "restaurant opening", "url": "https://www.google.com/alerts/feeds/14453255716137723784/12573505933789856657", "type": "google"}, {"name": "restaurant open soon", "url": "https://www.google.com/alerts/feeds/14453255716137723784/4441323395397190160", "type": "google"}, {"name": "restaurant opening soon", "url": "https://www.google.com/alerts/feeds/01056373355710472933/13281333592047130806", "type": "google"}, {"name": "restaurant sign lease", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5126161069635492598", "type": "google"}, {"name": "restaurant debut", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8573873082036518192", "type": "google"}, {"name": "restaurant new spots", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8834655103226671792", "type": "google"}, {"name": "restaurant new location", "url": "https://www.google.com/alerts/feeds/14453255716137723784/15937852298143138002", "type": "google"}, {"name": "restaurant another location", "url": "https://www.google.com/alerts/feeds/14453255716137723784/4715005523514208227", "type": "google"}, {"name": "second restaurant open", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2766045171956982817", "type": "google"}, {"name": "restaurant coming to", "url": "https://www.google.com/alerts/feeds/14453255716137723784/4589024498410374605", "type": "google"}, {"name": "now hiring restaurant", "url": "https://www.google.com/alerts/feeds/14453255716137723784/7524985625714946689", "type": "google"}, {"name": "restaurant job fair", "url": "https://www.google.com/alerts/feeds/14453255716137723784/4783798598907184871", "type": "google"}, {"name": "restaurant soft opening 2025", "url": "https://www.google.com/alerts/feeds/14453255716137723784/13512823969610652564", "type": "google"}, {"name": "restaurant permit filed", "url": "https://google.com/alerts/feeds/14453255716137723784/16856863385507748462", "type": "google"}, {"name": "restaurant liquor license", "url": "https://www.google.com/alerts/feeds/14453255716137723784/8829305381149378778", "type": "google"}, {"name": "restaurant lease signed", "url": "https://www.google.com/alerts/feeds/14453255716137723784/5204162851306534593", "type": "google"}, {"name": "restaurant groundbreaking", "url": "https://www.google.com/alerts/feeds/14453255716137723784/2507507732929226361", "type": "google"}, {"name": "restaurant renovation", "url": "https://www.google.com/alerts/feeds/14453255716137723784/3029673828230358427", "type": "google"}, {"name": "Texas Restaurant Signs Lease", "url": "https://www.google.com/alerts/feeds/14453255716137723784/16105551133591885866", "type": "google"}, ] # ============================================================================= # COMBINE ALL FEEDS - Priority order: WhatNow > Eater > Industry > Google # ============================================================================= RSS_FEEDS = WHATNOW_FEEDS + EATER_FEEDS + INDUSTRY_FEEDS + GOOGLE_ALERTS_FEEDS # ============================================================================= # BLOCKED SOURCES - Don't even try to scrape these (paywall/bot wall) # ============================================================================= BLOCKED_DOMAINS = [ "msn.com", "pressreader.com", "youtube.com", "facebook.com", "instagram.com", "twitter.com", "x.com", "bizjournals.com", "sfchronicle.com", "nytimes.com", "wsj.com", "bloomberg.com", "latimes.com", "washingtonpost.com", "bostonglobe.com", "dallasnews.com", "houstonchronicle.com", "denverpost.com", "startribune.com", "inquirer.com", "seattletimes.com", "chicagotribune.com", "miamiherald.com", "kansascity.com", "oregonlive.com", "mysanantonio.com", "expressnews.com", ] # ============================================================================= # EARLY FAIL PATTERNS - Reject at title level BEFORE scraping # ============================================================================= TITLE_FAIL_PATTERNS = [ # Retrospective / past openings r"opened in 20\d\d", r"that opened", r"best new.*of 20\d\d", r"restaurants of 20\d\d", r"year in review", r"happy hellos", r"sad goodbyes", r"best of 20\d\d", r"top \d+ restaurants of", r"favorites.*20\d\d", # Closures / negative events r"closed", r"closing", r"shut down", r"shuttered", r"fire damage", r"destroyed", r"bankruptcy", # Holiday hours / guides (not openings) r"open on new year", r"open for brunch", r"hours for", r"ring in the new year", r"where to eat", r"best restaurants in", r"top \d+ restaurants", r"guide to", r"food and drink stories", # Non-restaurant content r"hotel opening", r"real estate", r"million homes", ] # ============================================================================= # VALID US STATES # ============================================================================= US_STATES = [ 'AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA', 'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD', 'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ', 'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC', 'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY', 'DC' ] # ============================================================================= # PROCESSING LIMITS # ============================================================================= MAX_ARTICLES_PER_RUN = 1500 REQUEST_DELAY_SECONDS = 1 SCRAPE_TIMEOUT_SECONDS = 10 # ============================================================================= # OUTPUT # ============================================================================= LOG_FILE = f"scraper3_{datetime.now().strftime('%Y%m%d')}.log" # ============================================================================= # FEED STATS # ============================================================================= if __name__ == "__main__": print(f"Total feeds configured: {len(RSS_FEEDS)}") print(f" - WhatNow: {len(WHATNOW_FEEDS)}") print(f" - Eater: {len(EATER_FEEDS)}") print(f" - Industry: {len(INDUSTRY_FEEDS)}") print(f" - Google Alerts: {len(GOOGLE_ALERTS_FEEDS)}") print(f"\nBlocked domains: {len(BLOCKED_DOMAINS)}")