khushalps
7c9d094db8
git-subtree-dir: utils_v2 git-subtree-split: 0fdc939cad796fa02eb9d1f817c53d28cf8c2364
246 lines
9.4 KiB
Python
246 lines
9.4 KiB
Python
"""
|
|
|
|
AUTHOR:
|
|
|
|
Khushal P Soonderji
|
|
|
|
DATE:
|
|
|
|
Saturday, 18th May, 2024.
|
|
|
|
OBJECTIVE:
|
|
|
|
To provide a quick set of functions to work with fuzzy logic.
|
|
|
|
REFERENCES:
|
|
|
|
N/A
|
|
|
|
DOWNLOADS:
|
|
|
|
N/A
|
|
|
|
"""
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** IMPORT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# To make sibling directories accessible for imports:
|
|
import sys
|
|
sys.path.append(".")
|
|
sys.path.append("..")
|
|
|
|
# To apply fuzzy logic:
|
|
from thefuzz import fuzz, process
|
|
|
|
# To work with datatypes:
|
|
from typing import List
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MACROS / ONE-TIME INIT ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** VARIABLES ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
# --- Nothing Yet
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** FUNCTIONS ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
def get_best_match(
|
|
target: str,
|
|
choices: list[str],
|
|
threshold: float = 0.70,
|
|
partial: bool = False
|
|
) -> str | None:
|
|
|
|
"""
|
|
Given a list of options, this finds the best match to the target string as long as it fits within the similarity
|
|
threshold.
|
|
:param target: The string whose closest match needs to be found.
|
|
:param choices: The list of options to match the target against.
|
|
:param threshold: The absolute lowest similarity value to consider a match. Ranges from 0 to 1.
|
|
:param partial: If partial string matches are allowed.
|
|
:return: The string from the list of choices that is the closes match if it fits within the threshold, or None.
|
|
"""
|
|
|
|
# The input needs to be a string:
|
|
if not isinstance(target, str): return None
|
|
|
|
# Pick a scoring mechanism based on whether, or not, partial matches are allowed:
|
|
if partial: scorer = fuzz.partial_token_sort_ratio
|
|
else: scorer = fuzz.ratio
|
|
|
|
# Find the best match:
|
|
result = process.extractOne(
|
|
target,
|
|
choices,
|
|
score_cutoff = threshold * 100,
|
|
scorer = scorer
|
|
)
|
|
|
|
# Done here:
|
|
return result[0] if result else None
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def get_match_score(
|
|
target: str,
|
|
choice: str,
|
|
partial: bool = False
|
|
) -> float:
|
|
|
|
"""
|
|
Given a target and a choice to be considered, what is the similarity score of the choice to the target.
|
|
:param target: The string whose closest match needs to be found.
|
|
:param choice: The string that you want to test the target against.
|
|
:param partial: If partial string matches are allowed.
|
|
:return: The similarity score between the target and the choice (option) in the range from 0 to 1.
|
|
"""
|
|
|
|
# The input needs to be a string:
|
|
if not isinstance(target, str): return 0.0
|
|
|
|
# Pick a scoring mechanism based on whether, or not, partial matches are allowed:
|
|
if partial: scorer = fuzz.partial_token_sort_ratio
|
|
else: scorer = fuzz.ratio
|
|
|
|
# Find the best match with not threshold cut-off:
|
|
result = process.extractOne(
|
|
target,
|
|
choices = [choice],
|
|
score_cutoff = 0.0,
|
|
scorer = scorer
|
|
)
|
|
|
|
# Done here:
|
|
return result[1] / 100.0 if result else 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def rank(
|
|
target: str,
|
|
choices: List[str],
|
|
partial: bool = False
|
|
) -> List[dict] | None:
|
|
|
|
"""
|
|
To rank the similarity of all the options against the target.
|
|
:param target: The string against which all options need to be matched.
|
|
:param choices: The list of strings that need to be ranked for similarity with the target.
|
|
:param partial: If partial string matches are allowed.
|
|
:return: A list of all the choices with their similarity scores sorted from most to least similar.
|
|
"""
|
|
|
|
# The input needs to be a string:
|
|
if not isinstance(target, str): return None
|
|
|
|
# Pick a scoring mechanism based on whether, or not, partial matches are allowed:
|
|
if partial: scorer = fuzz.partial_token_sort_ratio
|
|
else: scorer = fuzz.ratio
|
|
|
|
# Match the target against all the options:
|
|
result = process.extract(
|
|
target,
|
|
choices,
|
|
limit = len(choices),
|
|
scorer = scorer
|
|
)
|
|
|
|
# Done here:
|
|
result = [{"choice": r[0], "score": r[1] / 100.0} for r in result]
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------------------------------------------------
|
|
|
|
|
|
def match(
|
|
targets: List[str],
|
|
choices: List[str],
|
|
threshold: float = 0.7,
|
|
partial: bool = False,
|
|
allow_null: bool = False
|
|
) -> dict | None:
|
|
|
|
"""
|
|
Find the best matches for each target from the list of choices such that each choice is used not more than once.
|
|
USE CASE: When you have a set of columns in a spreadsheet (choices) and you need to match them against a set of
|
|
expected values (targets). In such a case, if you absolutely need a match for every expected column
|
|
(target), you must set 'allow_null' to False. This could be useful in cases like bank statements.
|
|
:param targets: The list of string whose best matches must be found.
|
|
:param choices: The set of option strings that must be paired to the targets.
|
|
:param threshold: The absolute lowest similarity value to consider a match. Ranges from 0 to 1.
|
|
:param partial: If partial string matches are allowed.
|
|
:param allow_null: If set to False, each target must have a match and even one non-match will void the whole process
|
|
and return None. If set to True targets need not have matches.
|
|
:return: A dictionary of the best match for each target from the list of choices such that each choice is used just
|
|
once.
|
|
"""
|
|
|
|
# Start with basic variables:
|
|
all_rankings = []
|
|
all_matches = {target: None for target in targets}
|
|
|
|
# Match every choice against every target:
|
|
for target in targets:
|
|
rankings_for_target = rank(target, choices, partial = partial) or []
|
|
for r in rankings_for_target: r["target"] = target
|
|
all_rankings += rankings_for_target
|
|
|
|
# Sort everything based entirely on the final scores of similarity
|
|
# from most to least similar (descending order):
|
|
all_rankings = sorted(all_rankings, key = lambda x: x["score"], reverse = True)
|
|
|
|
# Now pick the target of the absolute best match,
|
|
# and keep eliminating the best match for each target:
|
|
while len(all_rankings) > 0:
|
|
top_ranking = all_rankings[0]
|
|
if top_ranking["score"] > threshold:
|
|
all_matches[top_ranking["target"]] = top_ranking["choice"]
|
|
elif not allow_null:
|
|
all_matches = None
|
|
break
|
|
all_rankings = [r for r in all_rankings if r["choice"] != top_ranking["choice"]]
|
|
|
|
# Done here:
|
|
return all_matches
|
|
|
|
|
|
# *****************************************************************************************************************
|
|
# ***** ****
|
|
# *** MAIN PROGRAM ***
|
|
# ***** ****
|
|
# *****************************************************************************************************************
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
pass
|