Revision control
Copy as Markdown
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
*/
use rusqlite::named_params;
use serde::Deserialize;
use sql_support::ConnExt;
use std::{cmp::Ordering, collections::HashSet};
use crate::{
config::SuggestProviderConfig,
db::{
KeywordInsertStatement, KeywordMetricsInsertStatement, SuggestDao,
SuggestionInsertStatement, DEFAULT_SUGGESTION_SCORE,
},
geoname::{GeonameMatch, GeonameType},
metrics::MetricsContext,
provider::SuggestionProvider,
rs::{Client, Record, SuggestRecordId},
store::SuggestStoreInner,
suggestion::Suggestion,
util::filter_map_chunks,
Result, SuggestionQuery,
};
#[derive(Clone, Debug, Deserialize)]
pub(crate) struct DownloadedWeatherAttachment {
/// Weather keywords.
pub keywords: Vec<String>,
/// Threshold for weather keyword prefix matching when a weather keyword is
/// the first term in a query. `None` means prefix matching is disabled and
/// weather keywords must be typed in full when they are first in the query.
/// This threshold does not apply to city and region names. If there are
/// multiple weather records, we use the `min_keyword_length` in the most
/// recently ingested record.
pub min_keyword_length: Option<i32>,
/// Score for weather suggestions. If there are multiple weather records, we
/// use the `score` from the most recently ingested record.
pub score: Option<f64>,
/// The max length of all keywords in the attachment. Used for keyword
/// metrics. We pre-compute this to avoid doing duplicate work on all user's
/// machines.
pub max_keyword_length: u32,
/// The max word count of all keywords in the attachment. Used for keyword
/// metrics. We pre-compute this to avoid doing duplicate work on all user's
/// machines.
pub max_keyword_word_count: u32,
}
/// This data is used to service every query handled by the weather provider, so
/// we cache it from the DB.
#[derive(Debug, Default)]
pub struct WeatherCache {
/// Cached value of the same name from `SuggestProviderConfig::Weather`.
min_keyword_length: i32,
/// Cached value of the same name from `SuggestProviderConfig::Weather`.
score: f64,
/// Max length of all weather keywords.
max_keyword_length: usize,
/// Max word count across all weather keywords.
max_keyword_word_count: usize,
}
impl SuggestDao<'_> {
/// Fetches weather suggestions.
pub fn fetch_weather_suggestions(&self, query: &SuggestionQuery) -> Result<Vec<Suggestion>> {
// We'll just stipulate we won't support tiny queries in order to avoid
// a bunch of work when the user starts typing a query.
if query.keyword.len() < 3 {
return Ok(vec![]);
}
// The first step in parsing the query is lowercasing and splitting it
// into words. We want to avoid that work for strings that are so long
// they can't possibly match. The longest possible weather query is two
// geonames + one weather keyword + at least two spaces between those
// three components, say, 10 extra characters total for spaces and
// punctuation. There's no point in an analogous min length check since
// weather suggestions can be matched on city alone and many city names
// are only a few characters long ("nyc").
let g_cache = self.geoname_cache();
let w_cache = self.weather_cache();
let max_query_len = 2 * g_cache.max_name_length + w_cache.max_keyword_length + 10;
if max_query_len < query.keyword.len() {
return Ok(vec![]);
}
let max_chunk_size =
std::cmp::max(g_cache.max_name_word_count, w_cache.max_keyword_word_count);
// Lowercase, strip punctuation, and split the query into words.
let kw_lower = query.keyword.to_lowercase();
let words: Vec<_> = kw_lower
.split_whitespace()
.flat_map(|w| {
w.split(|c| !char::is_alphabetic(c))
.filter(|s| !s.is_empty())
})
.collect();
let mut matches =
// Step 2: Parse the query words into a list of token paths.
filter_map_chunks::<Token>(&words, max_chunk_size, |chunk, chunk_i, path| {
// Match the chunk to token types that haven't already been matched
// in this path. `all_tokens` will remain `None` until a token is
// matched.
let mut all_tokens: Option<Vec<Token>> = None;
for tt in [
TokenType::City,
TokenType::Region,
TokenType::WeatherKeyword,
] {
if !path.iter().any(|t| t.token_type() == tt) {
let mut tokens = self.match_weather_tokens(tt, path, chunk, chunk_i == 0)?;
if !tokens.is_empty() {
let mut ts = all_tokens.take().unwrap_or_default();
ts.append(&mut tokens);
all_tokens.replace(ts);
}
}
}
// If no tokens were matched, `all_tokens` will be `None`.
Ok(all_tokens)
})?
.into_iter()
// Step 3: Map each token path to a city-region-keyword tuple (each
// optional). Paths are vecs, so they're ordered, so we may end up
// with duplicate tuples after this step. e.g., the paths
// `[<Waterloo IA>, <IA>]` and `[<IA>, <Waterloo IA>]` map to the
// same `(<Waterloo IA>, <IA>, None)` tuple.
.map(|path| {
path.into_iter()
.fold((None, None, None), |mut match_tuple, token| {
match token {
Token::City(c) => {
match_tuple.0 = Some(c);
}
Token::Region(r) => {
match_tuple.1 = Some(r);
}
Token::WeatherKeyword(kw) => {
match_tuple.2 = Some(kw);
}
}
match_tuple
})
})
// Step 4: Discard tuples that don't have the right combination of
// tokens or that are otherwise invalid. Along with step 2, this is
// the core of the matching logic. In general, allow a tuple if it
// has (a) a city name typed in full or (b) a weather keyword at
// least as long as the config's min keyword length, since that
// indicates a weather intent.
.filter(|(city_match, region_match, kw_match)| {
match (city_match, region_match, kw_match) {
(None, None, Some(_)) => true,
(None, _, None) | (None, Some(_), Some(_)) => false,
(Some(city), region, kw) => {
(city.match_type.is_name() && !city.prefix)
// Allow city abbreviations without a weather
// keyword but only if the region was typed in full.
|| (city.match_type.is_abbreviation()
&& !city.prefix
&& region.as_ref().map(|r| !r.prefix).unwrap_or(false))
|| kw.as_ref().map(|k| k.is_min_keyword_length).unwrap_or(false)
}
}
})
// Step 5: Map each tuple to a city-region tuple: Convert geoname
// matches to their `Geoname` values and discard keywords.
// Discarding keywords is important because we'll collect the tuples
// in a set in the next step in order to dedupe city-regions.
.map(|(city, region, _)| {
(city.map(|c| c.geoname), region.map(|r| r.geoname))
})
// Step 6: Dedupe the city-regions by collecting them in a set.
.collect::<HashSet<_>>()
.into_iter()
.collect::<Vec<_>>();
// Sort the matches so cities with larger populations are first.
matches.sort_by(
|(city1, region1), (city2, region2)| match (&city1, &city2) {
(Some(_), None) => Ordering::Less,
(None, Some(_)) => Ordering::Greater,
(Some(c1), Some(c2)) => c2.population.cmp(&c1.population),
(None, None) => match (®ion1, ®ion2) {
(Some(_), None) => Ordering::Less,
(None, Some(_)) => Ordering::Greater,
(Some(r1), Some(r2)) => r2.population.cmp(&r1.population),
(None, None) => Ordering::Equal,
},
},
);
// Finally, map matches to suggestions.
Ok(matches
.iter()
.map(|(city, _)| Suggestion::Weather {
city: city.as_ref().map(|c| c.name.clone()),
region: city.as_ref().map(|c| c.admin1_code.clone()),
country: city.as_ref().map(|c| c.country_code.clone()),
latitude: city.as_ref().map(|c| c.latitude),
longitude: city.as_ref().map(|c| c.longitude),
score: w_cache.score,
})
.collect())
}
fn match_weather_tokens(
&self,
token_type: TokenType,
path: &[Token],
candidate: &str,
is_first_chunk: bool,
) -> Result<Vec<Token>> {
match token_type {
TokenType::City => {
// Fetch matching cities, and filter them to regions we've
// already matched in this path.
let regions: Vec<_> = path
.iter()
.filter_map(|t| t.region().map(|m| &m.geoname))
.collect();
Ok(self
.fetch_geonames(
candidate,
!is_first_chunk,
Some(GeonameType::City),
if regions.is_empty() {
None
} else {
Some(regions)
},
)?
.into_iter()
.map(Token::City)
.collect())
}
TokenType::Region => {
// Fetch matching regions, and filter them to cities we've
// already matched in this patch.
let cities: Vec<_> = path
.iter()
.filter_map(|t| t.city().map(|m| &m.geoname))
.collect();
Ok(self
.fetch_geonames(
candidate,
!is_first_chunk,
Some(GeonameType::Region),
if cities.is_empty() {
None
} else {
Some(cities)
},
)?
.into_iter()
.map(Token::Region)
.collect())
}
TokenType::WeatherKeyword => {
// Fetch matching keywords. `min_keyword_length == 0` in the
// config means that the config doesn't allow prefix matching.
// `min_keyword_length > 0` means that the keyword must be at
// least that long when there's not already a city name present
// in the query.
let len = self.weather_cache().min_keyword_length;
if is_first_chunk && (candidate.len() as i32) < len {
// The candidate is the first term in the query and it's too
// short.
Ok(vec![])
} else {
// Do arbitrary prefix matching if the candidate isn't the
// first term in the query or if the config allows prefix
// matching.
Ok(self
.match_weather_keywords(candidate, !is_first_chunk || len > 0)?
.into_iter()
.map(|keyword| {
Token::WeatherKeyword(WeatherKeywordMatch {
keyword,
is_min_keyword_length: (len as usize) <= candidate.len(),
})
})
.collect())
}
}
}
}
fn match_weather_keywords(&self, candidate: &str, prefix: bool) -> Result<Vec<String>> {
self.conn.query_rows_and_then_cached(
r#"
SELECT
k.keyword,
s.score,
k.keyword != :keyword AS matched_prefix
FROM
suggestions s
JOIN
keywords k
ON k.suggestion_id = s.id
WHERE
s.provider = :provider
AND (
CASE :prefix WHEN FALSE THEN k.keyword = :keyword
ELSE (k.keyword BETWEEN :keyword AND :keyword || X'FFFF') END
)
"#,
named_params! {
":prefix": prefix,
":keyword": candidate,
":provider": SuggestionProvider::Weather
},
|row| -> Result<String> { Ok(row.get("keyword")?) },
)
}
/// Inserts weather suggestions data into the database.
fn insert_weather_data(
&mut self,
record_id: &SuggestRecordId,
attachments: &[DownloadedWeatherAttachment],
) -> Result<()> {
self.scope.err_if_interrupted()?;
let mut suggestion_insert = SuggestionInsertStatement::new(self.conn)?;
let mut keyword_insert = KeywordInsertStatement::new(self.conn)?;
let mut keyword_metrics_insert = KeywordMetricsInsertStatement::new(self.conn)?;
let mut max_len = 0;
let mut max_word_count = 0;
for attach in attachments {
let suggestion_id = suggestion_insert.execute(
record_id,
"",
"",
attach.score.unwrap_or(DEFAULT_SUGGESTION_SCORE),
SuggestionProvider::Weather,
)?;
for (i, keyword) in attach.keywords.iter().enumerate() {
keyword_insert.execute(suggestion_id, keyword, None, i)?;
}
self.put_provider_config(SuggestionProvider::Weather, &attach.into())?;
max_len = std::cmp::max(max_len, attach.max_keyword_length as usize);
max_word_count = std::cmp::max(max_word_count, attach.max_keyword_word_count as usize);
}
// Update keyword metrics.
keyword_metrics_insert.execute(
record_id,
SuggestionProvider::Weather,
max_len,
max_word_count,
)?;
// We just made some insertions that might invalidate the data in the
// cache. Clear it so it's repopulated the next time it's accessed.
self.weather_cache.take();
Ok(())
}
fn weather_cache(&self) -> &WeatherCache {
self.weather_cache.get_or_init(|| {
let mut cache = WeatherCache::default();
// keyword metrics
if let Ok((len, word_count)) = self.conn.query_row_and_then(
r#"
SELECT
max(max_length) AS len, max(max_word_count) AS word_count
FROM
keywords_metrics
WHERE
provider = :provider
"#,
named_params! {
":provider": SuggestionProvider::Weather
},
|row| -> Result<(usize, usize)> { Ok((row.get("len")?, row.get("word_count")?)) },
) {
cache.max_keyword_length = len;
cache.max_keyword_word_count = word_count;
}
// provider config
if let Ok(Some(SuggestProviderConfig::Weather {
score,
min_keyword_length,
})) = self.get_provider_config(SuggestionProvider::Weather)
{
cache.min_keyword_length = min_keyword_length;
cache.score = score;
}
cache
})
}
}
impl<S> SuggestStoreInner<S>
where
S: Client,
{
/// Inserts a weather record into the database.
pub fn process_weather_record(
&self,
dao: &mut SuggestDao,
record: &Record,
context: &mut MetricsContext,
) -> Result<()> {
self.download_attachment(dao, record, context, |dao, record_id, data| {
dao.insert_weather_data(record_id, data)
})
}
}
impl From<&DownloadedWeatherAttachment> for SuggestProviderConfig {
fn from(a: &DownloadedWeatherAttachment) -> Self {
Self::Weather {
score: a.score.unwrap_or(DEFAULT_SUGGESTION_SCORE),
min_keyword_length: a.min_keyword_length.unwrap_or(0),
}
}
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
enum TokenType {
City,
Region,
WeatherKeyword,
}
#[derive(Clone, Debug)]
enum Token {
City(GeonameMatch),
Region(GeonameMatch),
WeatherKeyword(WeatherKeywordMatch),
}
impl Token {
fn city(&self) -> Option<&GeonameMatch> {
match self {
Self::City(g) => Some(g),
_ => None,
}
}
fn region(&self) -> Option<&GeonameMatch> {
match self {
Self::Region(g) => Some(g),
_ => None,
}
}
fn token_type(&self) -> TokenType {
match self {
Self::City(_) => TokenType::City,
Self::Region(_) => TokenType::Region,
Self::WeatherKeyword(_) => TokenType::WeatherKeyword,
}
}
}
#[derive(Clone, Debug, Default, Eq, Hash, PartialEq)]
struct WeatherKeywordMatch {
keyword: String,
is_min_keyword_length: bool,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
geoname, geoname::Geoname, store::tests::TestStore, testing::*, SuggestIngestionConstraints,
};
impl From<Geoname> for Suggestion {
fn from(g: Geoname) -> Self {
Suggestion::Weather {
city: Some(g.name),
region: Some(g.admin1_code),
country: Some(g.country_code),
latitude: Some(g.latitude),
longitude: Some(g.longitude),
score: 0.24,
}
}
}
#[test]
fn weather_provider_config() -> anyhow::Result<()> {
before_each();
let store = TestStore::new(MockRemoteSettingsClient::default().with_record(
"weather",
"weather-1",
json!({
"min_keyword_length": 3,
"keywords": ["ab", "xyz", "weather"],
"max_keyword_length": "weather".len(),
"max_keyword_word_count": 1,
"score": 0.24
}),
));
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
assert_eq!(
store.fetch_provider_config(SuggestionProvider::Weather),
Some(SuggestProviderConfig::Weather {
score: 0.24,
min_keyword_length: 3,
})
);
Ok(())
}
#[test]
fn weather_keywords_prefixes_allowed() -> anyhow::Result<()> {
before_each();
let store = TestStore::new(MockRemoteSettingsClient::default().with_record(
"weather",
"weather-1",
json!({
// min_keyword_length > 0 means prefixes are allowed.
"min_keyword_length": 5,
"keywords": ["ab", "xyz", "cdefg", "weather"],
"max_keyword_length": "weather".len(),
"max_keyword_word_count": 1,
"score": 0.24
}),
));
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
let no_matches = [
// doesn't match any keyword
"ab123",
"123ab",
"xyz12",
"12xyz",
"xcdefg",
"cdefgx",
"x cdefg",
"cdefg x",
"weatherx",
"xweather",
"xweat",
"weatx",
"x weather",
" weather x",
"weather foo",
"foo weather",
// too short
"ab",
"xyz",
"cdef",
"we",
"wea",
"weat",
];
for q in no_matches {
assert_eq!(store.fetch_suggestions(SuggestionQuery::weather(q)), vec![]);
}
let matches = [
"cdefg",
"weath",
"weathe",
"weather",
"WeAtHeR",
" weather ",
];
for q in matches {
assert_eq!(
store.fetch_suggestions(SuggestionQuery::weather(q)),
vec![Suggestion::Weather {
score: 0.24,
city: None,
region: None,
country: None,
latitude: None,
longitude: None,
},]
);
}
Ok(())
}
#[test]
fn weather_keywords_prefixes_not_allowed() -> anyhow::Result<()> {
before_each();
let store = TestStore::new(MockRemoteSettingsClient::default().with_record(
"weather",
"weather-1",
json!({
// min_keyword_length == 0 means prefixes are not allowed.
"min_keyword_length": 0,
"keywords": ["weather"],
"max_keyword_length": "weather".len(),
"max_keyword_word_count": 1,
"score": 0.24
}),
));
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
let no_matches = ["wea", "weat", "weath", "weathe"];
for q in no_matches {
assert_eq!(store.fetch_suggestions(SuggestionQuery::weather(q)), vec![]);
}
let matches = ["weather", "WeAtHeR", " weather "];
for q in matches {
assert_eq!(
store.fetch_suggestions(SuggestionQuery::weather(q)),
vec![Suggestion::Weather {
score: 0.24,
city: None,
region: None,
country: None,
latitude: None,
longitude: None,
},]
);
}
Ok(())
}
#[test]
fn cities_and_regions() -> anyhow::Result<()> {
before_each();
let mut store = geoname::tests::new_test_store();
store.client_mut().add_record(
"weather",
"weather-1",
json!({
// Include a keyword that's a prefix of another keyword --
// "weather" and "weather near me" -- so that when a test
// matches both we can verify only one suggestion is returned,
// not two.
"keywords": ["ab", "xyz", "weather", "weather near me"],
"min_keyword_length": 5,
"max_keyword_length": "weather".len(),
"max_keyword_word_count": 1,
"score": 0.24
}),
);
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
let tests: &[(&str, Vec<Suggestion>)] = &[
(
"act",
vec![],
),
(
"act w",
vec![],
),
(
"act we",
vec![],
),
(
"act wea",
vec![],
),
(
"act weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"act weath",
vec![geoname::tests::waco().into()],
),
(
"act weathe",
vec![geoname::tests::waco().into()],
),
(
"act weather",
vec![geoname::tests::waco().into()],
),
(
"weather a",
// The made-up long-name city starts with A.
vec![geoname::tests::long_name_city().into()],
),
(
"weather ac",
vec![],
),
(
"weather act",
vec![geoname::tests::waco().into()],
),
(
"act t",
vec![],
),
(
"act tx",
vec![],
),
(
"act tx w",
vec![],
),
(
"act tx we",
vec![],
),
(
"act tx wea",
vec![],
),
(
"act tx weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"act tx weath",
vec![geoname::tests::waco().into()],
),
(
"act tx weathe",
vec![geoname::tests::waco().into()],
),
(
"act tx weather",
vec![geoname::tests::waco().into()],
),
(
"tx a",
vec![],
),
(
"tx ac",
vec![],
),
(
"tx act",
vec![],
),
(
"tx act w",
vec![],
),
(
"tx act we",
vec![],
),
(
"tx act wea",
vec![],
),
(
"tx act weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"tx act weath",
vec![geoname::tests::waco().into()],
),
(
"tx act weathe",
vec![geoname::tests::waco().into()],
),
(
"tx act weather",
vec![geoname::tests::waco().into()],
),
(
"act te",
vec![],
),
(
"act tex",
vec![],
),
(
"act texa",
vec![],
),
(
"act texas",
vec![],
),
(
"act texas w",
vec![],
),
(
"act texas we",
vec![],
),
(
"act texas wea",
vec![],
),
(
"act texas weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"act texas weath",
vec![geoname::tests::waco().into()],
),
(
"act texas weathe",
vec![geoname::tests::waco().into()],
),
(
"act texas weather",
vec![geoname::tests::waco().into()],
),
(
"texas a",
vec![],
),
(
"texas ac",
vec![],
),
(
"texas act",
vec![],
),
(
"texas act w",
vec![],
),
(
"texas act we",
vec![],
),
(
"texas act wea",
vec![],
),
(
"texas act weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"texas act weath",
vec![geoname::tests::waco().into()],
),
(
"texas act weathe",
vec![geoname::tests::waco().into()],
),
(
"texas act weather",
vec![geoname::tests::waco().into()],
),
(
"ia w",
vec![],
),
(
"ia wa",
vec![],
),
(
"ia wat",
vec![],
),
(
"ia wate",
vec![],
),
(
"ia water",
vec![],
),
(
"ia waterl",
vec![],
),
(
"ia waterlo",
vec![],
),
(
"waterloo",
vec![
// Waterloo, IA should be first since its population is
// larger than Waterloo, AL.
geoname::tests::waterloo_ia().into(),
geoname::tests::waterloo_al().into(),
],
),
(
"waterloo i",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo ia",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo io",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo iow",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo iowa",
vec![geoname::tests::waterloo_ia().into()],
),
(
"ia waterloo",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo al",
vec![geoname::tests::waterloo_al().into()],
),
(
"al waterloo",
vec![geoname::tests::waterloo_al().into()],
),
("waterloo ia al", vec![]),
("waterloo ny", vec![]),
(
"ia",
vec![],
),
(
"iowa",
vec![],
),
(
"al",
vec![],
),
(
"alabama",
vec![],
),
(
"new york",
vec![geoname::tests::nyc().into()],
),
(
"new york new york",
vec![geoname::tests::nyc().into()],
),
(
"ny ny",
vec![geoname::tests::nyc().into()],
),
("ny ny ny", vec![]),
(
"ny n",
vec![],
),
(
"ny ne",
vec![],
),
(
"ny new",
vec![],
),
(
"ny new ",
vec![],
),
(
"ny new y",
vec![],
),
(
"ny new yo",
vec![],
),
(
"ny new yor",
vec![],
),
(
"ny new york",
vec![geoname::tests::nyc().into()],
),
(
"new york ny",
vec![geoname::tests::nyc().into()],
),
(
"weather ny",
vec![geoname::tests::nyc().into()],
),
(
"ny weather",
vec![geoname::tests::nyc().into()],
),
(
"ny w",
vec![],
),
(
"ny we",
vec![],
),
(
"ny wea",
vec![],
),
(
"ny weat",
vec![],
),
(
// `min_keyword_length` = 5, so there should be a match.
"ny weath",
vec![geoname::tests::nyc().into()],
),
(
"ny weathe",
vec![geoname::tests::nyc().into()],
),
(
"weather ny ny",
vec![geoname::tests::nyc().into()],
),
(
"ny weather ny",
vec![geoname::tests::nyc().into()],
),
(
"ny ny weather",
vec![geoname::tests::nyc().into()],
),
(
"rochester ny",
vec![geoname::tests::rochester().into()],
),
(
"ny rochester",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester weather ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester ny weather",
vec![geoname::tests::rochester().into()],
),
(
"weather ny rochester",
vec![geoname::tests::rochester().into()],
),
(
"ny weather rochester",
vec![geoname::tests::rochester().into()],
),
(
"ny rochester weather",
vec![geoname::tests::rochester().into()],
),
(
"weather new york",
vec![geoname::tests::nyc().into()],
),
(
"new weather york",
vec![],
),
(
"new york weather",
vec![geoname::tests::nyc().into()],
),
(
"weather new york new york",
vec![geoname::tests::nyc().into()],
),
(
"new york weather new york",
vec![geoname::tests::nyc().into()],
),
(
"new york new york weather",
vec![geoname::tests::nyc().into()],
),
(
"weather water",
vec![
geoname::tests::waterloo_ia().into(),
geoname::tests::waterloo_al().into(),
],
),
(
"waterloo w",
vec![
geoname::tests::waterloo_ia().into(),
geoname::tests::waterloo_al().into(),
],
),
("weather w w", vec![]),
("weather w water", vec![]),
("weather w waterloo", vec![]),
("weather water w", vec![]),
("weather waterloo water", vec![]),
("weather water water", vec![]),
("weather water waterloo", vec![]),
("waterloo foo", vec![]),
("waterloo weather foo", vec![]),
("foo waterloo", vec![]),
("foo waterloo weather", vec![]),
(
"ny",
vec![],
),
(
"nyc",
vec![],
),
(
"roc",
vec![],
),
(
"nyc ny",
vec![geoname::tests::nyc().into()],
),
(
"ny nyc",
vec![geoname::tests::nyc().into()],
),
(
"roc ny",
vec![],
),
(
"ny roc",
vec![],
),
(
"nyc weather",
vec![geoname::tests::nyc().into()],
),
(
"weather nyc",
vec![geoname::tests::nyc().into()],
),
(
"roc weather",
vec![geoname::tests::rochester().into()],
),
(
"weather roc",
vec![geoname::tests::rochester().into()],
),
(
geoname::tests::LONG_NAME,
vec![geoname::tests::long_name_city().into()],
),
(
" WaTeRlOo ",
vec![
geoname::tests::waterloo_ia().into(),
geoname::tests::waterloo_al().into(),
],
),
(
" waterloo ia",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo ia",
vec![geoname::tests::waterloo_ia().into()],
),
(
"waterloo ia ",
vec![geoname::tests::waterloo_ia().into()],
),
(
" waterloo ia ",
vec![geoname::tests::waterloo_ia().into()],
),
(
" new york weather",
vec![geoname::tests::nyc().into()],
),
(
"new york weather",
vec![geoname::tests::nyc().into()],
),
(
"new york weather",
vec![geoname::tests::nyc().into()],
),
(
"new york weather ",
vec![geoname::tests::nyc().into()],
),
(
"rochester,",
vec![geoname::tests::rochester().into()],
),
(
"rochester ,",
vec![geoname::tests::rochester().into()],
),
(
"rochester , ",
vec![geoname::tests::rochester().into()],
),
(
"rochester,ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester, ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester ,ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester , ny",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester,",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester, ",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester , ",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester,ny",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester, ny",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester ,ny",
vec![geoname::tests::rochester().into()],
),
(
"weather rochester , ny",
vec![geoname::tests::rochester().into()],
),
(
"rochester,weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester, weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester ,weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester , weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester,ny weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester, ny weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester ,ny weather",
vec![geoname::tests::rochester().into()],
),
(
"rochester , ny weather",
vec![geoname::tests::rochester().into()],
),
(
"new york,",
vec![geoname::tests::nyc().into()],
),
(
"new york ,",
vec![geoname::tests::nyc().into()],
),
(
"new york , ",
vec![geoname::tests::nyc().into()],
),
(
"new york,ny",
vec![geoname::tests::nyc().into()],
),
(
"new york, ny",
vec![geoname::tests::nyc().into()],
),
(
"new york ,ny",
vec![geoname::tests::nyc().into()],
),
(
"new york , ny",
vec![geoname::tests::nyc().into()],
),
(
"weather new york,ny",
vec![geoname::tests::nyc().into()],
),
(
"weather new york, ny",
vec![geoname::tests::nyc().into()],
),
(
"weather new york ,ny",
vec![geoname::tests::nyc().into()],
),
(
"weather new york , ny",
vec![geoname::tests::nyc().into()],
),
(
"new york,weather",
vec![geoname::tests::nyc().into()],
),
(
"new york, weather",
vec![geoname::tests::nyc().into()],
),
(
"new york ,weather",
vec![geoname::tests::nyc().into()],
),
(
"new york , weather",
vec![geoname::tests::nyc().into()],
),
(
"new york,ny weather",
vec![geoname::tests::nyc().into()],
),
(
"new york, ny weather",
vec![geoname::tests::nyc().into()],
),
(
"new york ,ny weather",
vec![geoname::tests::nyc().into()],
),
(
"new york , ny weather",
vec![geoname::tests::nyc().into()],
),
(
&format!("{} weather", geoname::tests::LONG_NAME),
vec![geoname::tests::long_name_city().into()],
),
(
&format!("weather {}", geoname::tests::LONG_NAME),
vec![geoname::tests::long_name_city().into()],
),
(
&format!("{} and some other words that don't match anything but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and some other words that don't match anything {} but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and some other words that don't match anything but that is neither here nor there {}", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("weather {} and some other words that don't match anything but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("{} weather and some other words that don't match anything but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("{} and some other words that don't match anything weather but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("{} and some other words that don't match anything but that is neither here nor there weather", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("weather and some other words that don't match anything {} but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("weather and some other words that don't match anything but that is neither here nor there {}", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and some other words that don't match anything weather {} but that is neither here nor there", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and some other words that don't match anything but that is neither here nor there weather {}", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("{} weather and then this also doesn't match anything down here", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("{} and then this also doesn't match anything down here weather", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and then this also doesn't match anything down here {} weather", geoname::tests::LONG_NAME),
vec![]
),
(
&format!("and then this also doesn't match anything down here weather {}", geoname::tests::LONG_NAME),
vec![]
),
];
for (query, expected_suggestions) in tests {
assert_eq!(
&store.fetch_suggestions(SuggestionQuery::weather(query)),
expected_suggestions,
"Query: {:?}",
query
);
}
Ok(())
}
#[test]
fn keywords_metrics() -> anyhow::Result<()> {
before_each();
// Add a couple of records with different metrics. We're just testing
// metrics so the other values don't matter.
let mut store = TestStore::new(
MockRemoteSettingsClient::default()
.with_record(
"weather",
"weather-0",
json!({
"max_keyword_length": 10,
"max_keyword_word_count": 5,
"min_keyword_length": 3,
"score": 0.24,
"keywords": []
}),
)
.with_record(
"weather",
"weather-1",
json!({
"max_keyword_length": 20,
"max_keyword_word_count": 2,
"min_keyword_length": 3,
"score": 0.24,
"keywords": []
}),
),
);
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
store.read(|dao| {
let cache = dao.weather_cache();
assert_eq!(cache.max_keyword_length, 20);
assert_eq!(cache.max_keyword_word_count, 5);
Ok(())
})?;
// Delete the first record. The metrics should change.
store
.client_mut()
.delete_record("quicksuggest", "weather-0");
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
store.read(|dao| {
let cache = dao.weather_cache();
assert_eq!(cache.max_keyword_length, 20);
assert_eq!(cache.max_keyword_word_count, 2);
Ok(())
})?;
// Add a new record. The metrics should change again.
store.client_mut().add_record(
"weather",
"weather-3",
json!({
"max_keyword_length": 15,
"max_keyword_word_count": 3,
"min_keyword_length": 3,
"score": 0.24,
"keywords": []
}),
);
store.ingest(SuggestIngestionConstraints {
providers: Some(vec![SuggestionProvider::Weather]),
..SuggestIngestionConstraints::all_providers()
});
store.read(|dao| {
let cache = dao.weather_cache();
assert_eq!(cache.max_keyword_length, 20);
assert_eq!(cache.max_keyword_word_count, 3);
Ok(())
})?;
Ok(())
}
}