---
title: "Regex search"
output: rmarkdown::html_vignette
vignette: >
  %\VignetteIndexEntry{Regex search}
  %\VignetteEngine{knitr::rmarkdown}
  %\VignetteEncoding{UTF-8}
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(
  eval = identical(Sys.getenv("TEXTPRESS_BUILD_LIVE_VIGNETTES"), "true"),
  message = FALSE,
  warning = FALSE,
  comment = "#>"
)
```

`search_regex()` matches patterns against a text corpus and returns results in KWIC format with inline highlighting. This vignette applies it to a generational-politics corpus with three patterns -- age ranges, numeric change expressions, and voter-mobilization language -- each match returned at the sentence level, ready for close reading or aggregation.

## Terms and RSS entries

Terms used to select relevant entries from political RSS feeds.

```{r search-terms}
search_terms <- c(
  "Gen Z", "Millennial", "Gen X", "Baby Boomer",
  "young voters", "older voters"
)
```


Fetch recent entries from the selected feeds, filter their title and description locally, and deduplicate the resulting URLs.

```{r web-urls, message=FALSE, warning=FALSE}
library(textpress)
library(dplyr)
library(DT)

politics_feeds <- textpress::rss_politics

web_urls <- textpress::fetch_rss(politics_feeds$url) |>
  filter(grepl(
    paste(search_terms, collapse = "|"),
    paste(title, description),
    ignore.case = TRUE
  )) |>
  select(-doc_id) |>
  distinct(url, .keep_all = TRUE)
```



## Web text and sentence split

`read_urls()` scrapes and parses the article text; `nlp_split_sentences()` segments each document into analysis-ready sentence rows.

```{r web-text}
web_text_list <- web_urls |>
  textpress::read_urls(cores = 6) 

web_text <- web_text_list$text |>
  mutate(doc_id = match(url, unique(url))) |>
  relocate(doc_id, .before = 1)
```


Segment each document into sentence rows.

```{r web-ss}
web_ss <- web_text |>
  textpress::nlp_split_sentences(by = c('doc_id', 'node_id'))
```



## Regex patterns

Three patterns: `age_range` matches explicit age constructions; `from_to` captures numeric change expressions; `energized` catches voter-mobilization language.

```{r patterns}
patterns <- list(
  energized = "\\b(?:energi[zs]ed|motivated|mobili[zs]ed|fired\\s+up|disillusioned|apathetic)\\b",
age_range = paste(
  "\\b(?:aged?(?:\\s+between)?|ages?)\\s+\\d{2,3}(?:(?:\\s*(?:[-–]|to|and)\\s*\\d{2,3})|(?:\\s+and\\s+(?:older|younger|over|under)))\\b",
  "\\bunder\\s+\\d{2}(?!\\d|,|%)\\b",
  "\\b\\d{2,3}(?!\\d|,|%)\\s*(?:\\+|and\\s+(?:older|younger|over|under))\\b",
  sep = "|"
),
  from_to   = "\\bfrom\\s+\\d+\\s+to\\s+\\d+\\b"
)
```



## Search results

### Search ~ `age_range`

Sentences containing explicit age ranges or constructions (e.g. *"adults aged 18 to 29"*, *"under 30"*, *"65+"*).

```{r out-age-range}
fs <- web_ss |> 
  
  textpress::search_regex(
    query = patterns$age_range,
    by = c('doc_id', 'node_id'),
    highlight = c('<span style="background:#a6cbe1;">', '</span>')
    ) |>
  
  distinct(text, .keep_all = TRUE) |>
  select(doc_id, node_id, pattern, text)

if (!is.null(fs) && nrow(fs) > 0) fs |> DT::datatable(rownames = FALSE, escape = FALSE)
```



### Search ~ `from_to`

Sentences with numeric change expressions (e.g. *"from 42 to 51 percent"*) -- useful for tracking shifts in polling or support figures.

```{r out-from-to}
fs1 <- web_ss |> 
  
  textpress::search_regex(
    query = patterns$from_to,
    by = c('doc_id', 'node_id'),
    highlight = c('<span style="background:#fdc4a8;">', '</span>')
    ) |>
  
  distinct(text, .keep_all = TRUE) |>
  select(doc_id, node_id, pattern, text)

if (!is.null(fs1) && nrow(fs1) > 0) fs1 |> DT::datatable(rownames = FALSE, escape = FALSE)
```

### Search ~ `energized`

Sentences with voter-mobilization language -- terms like *"energized"*, *"motivated"*, *"mobilized"*, *"disillusioned"*, *"apathetic"*.

```{r out-energized}
fs2 <- web_ss |>

  textpress::search_regex(
    query = patterns$energized,
    by = c('doc_id', 'node_id'),
    highlight = c('<span style="background:#c8e6c9;">', '</span>')
    ) |>

  distinct(text, .keep_all = TRUE) |>
  select(doc_id, node_id, pattern, text)

if (!is.null(fs2) && nrow(fs2) > 0) fs2 |> DT::datatable(rownames = FALSE, escape = FALSE)
```

## Summary

`search_regex()` is best for precise, expressive patterns -- age ranges, numeric constructions, multi-word sequences -- where you control the match exactly. Results come back sentence-level with inline highlighting. For broader vocabulary coverage without hand-crafting every variant, see the dictionary search vignette.