How I use LLMs and Firecrawl to dig into company culture

# company_culture_with_web_search.py
import logging
from dotenv import load_dotenv
from openai import OpenAI

URLS = [
    "https://cursor.com/blog/series-d",
    "https://cursor.com/blog/productivity",
    "https://cursor.com/blog/enterprise",
]

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

load_dotenv()

client = OpenAI()

base_prompt = """For each article I provide, do the following and present the results in a separate section per article:

- Author: (or "Unknown" if not stated)
- Date: publication date (or "Unknown")
- Role: author's role (job title or inferred role)
- Style: Style, tone, experience level, and inferred author psychology (1 sentence)
- Company culture: What does the article suggest about the company culture?  (1 sentence)
- Summary: summarize the article as 3 actionable items, each formatted as:

  - What: [max 12 words]
  - Why: [max 12 words]
  - Psychology: [max 12 words]

If information is missing or ambiguous, state that explicitly and briefly explain your inference, if any.
"""

prompt_with_urls = (
    "Perform this analysis for every article listed below.  Do not skip any of them.\n\n"
    + base_prompt
    + "\n\n"
    + "\n".join(URLS)
)

response = client.responses.create(
    model="gpt-5.1",
    tools=[{"type": "web_search"}],
    include=["web_search_call.action.sources"],
    input=prompt_with_urls,
)

with open("summaries_with_web_search.md", "w", encoding="utf-8") as f:
    f.write(response.output_text)

# Log URLs that aren't among the sources used by the web_search tool.
sources = []
for item in response.output:
    if item.type == "web_search_call":
        if action := getattr(item, "action", None):
            for s in getattr(action, "sources", []):
                if url := getattr(s, "url", None):
                    sources.append(url)

all_urls_included = True
for url in URLS:
    if url not in sources:
        all_urls_included = False
        logging.info(f"URL not included in web search sources: {url}")

if all_urls_included:
    logging.info("All URLs were included in web search sources.")

# company_culture.py
import os
import logging
from dotenv import load_dotenv
from firecrawl import Firecrawl
from openai import OpenAI

URLS = [
    "https://cursor.com/blog/series-d",
    "https://cursor.com/blog/productivity",
    "https://cursor.com/blog/enterprise",
]

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

load_dotenv()

# 1) Download articles in markdown format with Firecrawl
api_key = os.getenv("FIRECRAWL_API_KEY")
firecrawl = Firecrawl(api_key=api_key)

# Ensure articles.md is empty
with open("articles.md", "w", encoding="utf-8"):
    pass

with open("articles.md", "a", encoding="utf-8") as f:
    for url in URLS:
        logging.info(f"Downloading {url}")
        try:
            page = firecrawl.scrape(url, formats=["markdown"])
            f.write("<|article_begin|>\n")
            f.write(page.markdown)
            f.write("\n<|article_end|>\n\n")
        except Exception as e:
            logging.error(f"Downloading {url}: {e}")

# 2) Summarize, focusing on company culture

client = OpenAI()

base_prompt = """For each article I provide, do the following and present the results in a separate section per article:

- Author: (or "Unknown" if not stated)
- Date: publication date (or "Unknown")
- Role: author's role (job title or inferred role)
- Style: Style, tone, experience level, and inferred author psychology (1 sentence)
- Company culture: What does the article suggest about the company culture?  (1 sentence)
- Summary: summarize the article as 3 actionable items, each formatted as:

  - What: [max 12 words]
  - Why: [max 12 words]
  - Psychology: [max 12 words]

If information is missing or ambiguous, state that explicitly and briefly explain your inference, if any.
"""

with open("articles.md", "r", encoding="utf-8") as f:
    articles = f.read()

prompt_with_articles = (
    "Perform this analysis for every article in the following Markdown snippets.  Do not skip any.  Each article is wrapped in `<|article_begin|>...<|article_end|>`.\n\n"
    + base_prompt
    + "\n\n"
    + articles
)

response = client.responses.create(
    model="gpt-5.1",
    input=prompt_with_articles,
)

with open("summaries.md", "w", encoding="utf-8") as f:
    f.write(response.output_text)

To retrieve the blog posts with Bright Data:

1. Add the brightdata-sdk dependency:

   $ uv add brightdata-sdk

2. Add your Bright Data API key to the .env file:

   BRIGHTDATA_API_TOKEN=...

3. Replace the section "Download articles in markdown format with Firecrawl" in company_culture.md file with:

   from brightdata import bdclient
   
   api_token = os.getenv("BRIGHTDATA_API_KEY")
   brightdata = bdclient(api_token=api_token)
   
   # Ensure articles.md is empty
   with open("articles.md", "w", encoding="utf-8"):
       pass
   
   with open("articles.md", "a", encoding="utf-8") as f:
       for url in URLS:
           logging.info(f"Downloading {url}")
           try:
               page = brightdata.scrape(
                   url=url,
                   data_format="markdown",
                   async_request=False,
                   timeout=60,
               )
               f.write("<|article_begin|>\n")
               f.write(page)
               f.write("\n<|article_end|>\n\n")
           except Exception as e:
               logging.error(f"Downloading {url}: {e}")