Skip to content

Commit

Permalink
Rename some funcs.
Browse files Browse the repository at this point in the history
  • Loading branch information
xkww3n committed Dec 1, 2023
1 parent 3ad3e00 commit 8fb4a06
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 40 deletions.
26 changes: 13 additions & 13 deletions Tests/test_2_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,24 +24,24 @@ def test_is_domain(self):
assert not rule.is_domain(invalid_domain_b)
assert not rule.is_domain(ipv4addr)

def test_custom_convert_domain(self):
def test_load_domain(self):
test_src_path = Path("./src/custom_ruleset/")
test_conv_ruleset = ruleset.custom_convert(test_src_path/"domain.txt")
assert test_conv_ruleset == ruleset.RuleSet("Domain",
loaded_ruleset = ruleset.load(test_src_path / "domain.txt")
assert loaded_ruleset == ruleset.RuleSet("Domain",
[rule.Rule("DomainSuffix", "example.com"),
rule.Rule("DomainFull", "example.com")])

def test_custom_convert_ipcidr(self):
def test_load_ipcidr(self):
test_src_path = Path("./src/custom_ruleset/")
test_conv_ruleset = ruleset.custom_convert(test_src_path/"ipcidr.txt")
assert test_conv_ruleset == ruleset.RuleSet("IPCIDR",
loaded_ruleset = ruleset.load(test_src_path / "ipcidr.txt")
assert loaded_ruleset == ruleset.RuleSet("IPCIDR",
[rule.Rule("IPCIDR", "11.4.5.14"),
rule.Rule("IPCIDR6", "fc00:114::514")])

def test_custom_convert_combined(self):
def test_load_combined(self):
test_src_path = Path("./src/custom_ruleset/")
test_conv_ruleset = ruleset.custom_convert(test_src_path/"combined.txt")
assert test_conv_ruleset == ruleset.RuleSet("Combined",
loaded_ruleset = ruleset.load(test_src_path / "combined.txt")
assert loaded_ruleset == ruleset.RuleSet("Combined",
[rule.Rule("DomainFull", "example.com"),
rule.Rule("DomainSuffix", "example.com"),
rule.Rule("IPCIDR", "11.4.5.14"),
Expand All @@ -50,12 +50,12 @@ def test_custom_convert_combined(self):
def test_patch(self):
test_src_patch = Path("./src/patch/")
test_ruleset = ruleset.RuleSet("Domain", [rule.Rule("DomainFull", "example.com")])
ruleset.apply_patch(test_ruleset, "patch", test_src_patch)
ruleset.patch(test_ruleset, "patch", test_src_patch)
assert test_ruleset == ruleset.RuleSet("Domain", [rule.Rule("DomainSuffix", "example.com")])

def test_dump_domain(self):
test_dist = Path("./dists/")
ruleset_domain = ruleset.custom_convert(Path("./src/custom_ruleset/domain.txt"))
ruleset_domain = ruleset.load(Path("./src/custom_ruleset/domain.txt"))
ruleset.batch_dump(ruleset_domain, const.TARGETS, test_dist, "domain")

assert (test_dist/"text"/"domain.txt").exists()
Expand Down Expand Up @@ -91,7 +91,7 @@ def test_dump_domain(self):

def test_dump_ipcidr(self):
test_dist = Path("./dists/")
ruleset_ipcidr = ruleset.custom_convert(Path("./src/custom_ruleset/ipcidr.txt"))
ruleset_ipcidr = ruleset.load(Path("./src/custom_ruleset/ipcidr.txt"))
ruleset.batch_dump(ruleset_ipcidr, const.TARGETS, test_dist, "ipcidr")

assert not (test_dist/"text-plus"/"ipcidr.txt").exists()
Expand Down Expand Up @@ -120,7 +120,7 @@ def test_dump_ipcidr(self):

def test_dump_combined(self):
test_dist = Path("./dists/")
ruleset_combined = ruleset.custom_convert(Path("./src/custom_ruleset/combined.txt"))
ruleset_combined = ruleset.load(Path("./src/custom_ruleset/combined.txt"))
ruleset.batch_dump(ruleset_combined, const.TARGETS, test_dist, "combined")

assert not (test_dist/"text"/"combined.txt").exists()
Expand Down
2 changes: 1 addition & 1 deletion Utils/geosite.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def parse(src: set, excluded_imports=None, excluded_tags=None) -> ruleset.RuleSe
return ruleset_parsed


def batch_convert(categories: list, tools: list, exclusions=None) -> None:
def batch_gen(categories: list, tools: list, exclusions=None) -> None:
exclusions = [] if not exclusions else exclusions
for tool in tools:
for category in categories:
Expand Down
40 changes: 20 additions & 20 deletions Utils/ruleset.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,44 +101,44 @@ def dedup(self):
self.Payload = list_unique


def custom_convert(src: Path) -> RuleSet:
src_custom = open(src, mode="r", encoding="utf-8").read().splitlines()
def load(src: Path) -> RuleSet:
src_toload = open(src, mode="r", encoding="utf-8").read().splitlines()
try:
ruleset_type = src_custom[0].split("@")[1]
ruleset_type = src_toload[0].split("@")[1]
except IndexError:
logging.warning(f"File {src} doesn't have a valid type header, treat as domain type.")
ruleset_type = "Domain"
ruleset_converted = RuleSet(ruleset_type, [])
ruleset_loaded = RuleSet(ruleset_type, [])
match ruleset_type:
case "Domain":
for line in src_custom:
for line in src_toload:
if line.startswith("."):
ruleset_converted.add(Rule("DomainSuffix", line.strip(".")))
ruleset_loaded.add(Rule("DomainSuffix", line.strip(".")))
elif line and not line.startswith("#"):
ruleset_converted.add(Rule("DomainFull", line))
ruleset_loaded.add(Rule("DomainFull", line))
case "IPCIDR":
for line in src_custom:
for line in src_toload:
if line and not line.startswith("#"):
if ":" in line:
ruleset_converted.add(Rule("IPCIDR6", line))
ruleset_loaded.add(Rule("IPCIDR6", line))
else:
ruleset_converted.add(Rule("IPCIDR", line))
ruleset_loaded.add(Rule("IPCIDR", line))
case "Combined":
for line in src_custom:
for line in src_toload:
if line and not line.startswith("#"):
parsed = line.split(",")
match parsed[0]:
case "DOMAIN":
ruleset_converted.add(Rule("DomainFull", parsed[1]))
ruleset_loaded.add(Rule("DomainFull", parsed[1]))
case "DOMAIN-SUFFIX":
ruleset_converted.add(Rule("DomainSuffix", parsed[1]))
ruleset_loaded.add(Rule("DomainSuffix", parsed[1]))
case "IP-CIDR":
ruleset_converted.add(Rule("IPCIDR", parsed[1]))
ruleset_loaded.add(Rule("IPCIDR", parsed[1]))
case "IP-CIDR6":
ruleset_converted.add(Rule("IPCIDR6", parsed[1]))
ruleset_loaded.add(Rule("IPCIDR6", parsed[1]))
case _:
raise ValueError()
return ruleset_converted
return ruleset_loaded


def dump(src: RuleSet, target: str, dst: Path, filename: str) -> None:
Expand Down Expand Up @@ -276,17 +276,17 @@ def batch_dump(src: RuleSet, targets: list, dst_path: Path, filename: str) -> No
dump(src, target, dst_path/target, filename)


def apply_patch(src: RuleSet, name: str, override_patch_loc: Path = Path("")) -> RuleSet:
def patch(src: RuleSet, name: str, override_patch_loc: Path = Path("")) -> RuleSet:
try:
if override_patch_loc != Path(""):
patch = open(override_patch_loc/(name + ".txt"), mode="r").read().splitlines()
loaded_patch = open(override_patch_loc/(name + ".txt"), mode="r").read().splitlines()
else:
patch = open(const.PATH_SOURCE_PATCH/(name + ".txt"), mode="r").read().splitlines()
loaded_patch = open(const.PATH_SOURCE_PATCH/(name + ".txt"), mode="r").read().splitlines()
except FileNotFoundError:
logging.warning(f'Patch "{name + ".txt"}" not found.')
return src
logging.info(f'Start applying patch "{name + ".txt"}"')
for line in patch:
for line in loaded_patch:
if line.startswith("#"):
continue
parsed_line = line.split(":")
Expand Down
12 changes: 6 additions & 6 deletions generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,8 @@
ruleset_exclusions.add(domain_exclude)
logger.debug(f"{domain_exclude} is added to final exclude set.")

ruleset_rejections = ruleset.apply_patch(ruleset_rejections, "reject")
ruleset_exclusions = ruleset.apply_patch(ruleset_exclusions, "exclude")
ruleset_rejections = ruleset.patch(ruleset_rejections, "reject")
ruleset_exclusions = ruleset.patch(ruleset_exclusions, "exclude")

logger.info(f"Generated {len(ruleset_rejections)} reject rules.")
logger.info(f"Generated {len(ruleset_exclusions)} exclude rules.")
Expand All @@ -114,7 +114,7 @@
if any([item.Payload.endswith(os_tld) for os_tld in tld_overseas]):
ruleset_domestic.remove(item)
logger.debug(f"{item} removed for having a overseas TLD.")
ruleset_domestic = ruleset.apply_patch(ruleset_domestic, "domestic")
ruleset_domestic = ruleset.patch(ruleset_domestic, "domestic")

# Add all domestic TLDs to domestic rules, then perform deduplication.
src_domestic_tlds = set(open(const.PATH_SOURCE_V2FLY/"tld-cn", mode="r", encoding="utf-8").read().splitlines())
Expand Down Expand Up @@ -176,7 +176,7 @@
"github", # GitHub's domains are included in "microsoft", but its connectivity mostly isn't as high as Microsoft.
"bing", # Bing has a more restricted ver for Mainland China.
]
geosite.batch_convert(CATEGORIES, const.TARGETS, EXCLUSIONS)
geosite.batch_gen(CATEGORIES, const.TARGETS, EXCLUSIONS)

END_TIME = time_ns()
logger.info(f"Finished. Total time: {format((END_TIME - START_TIME) / 1e9, '.3f')}s\n")
Expand All @@ -188,15 +188,15 @@
for filename in list_file_custom:
if filename.is_file():
logger.debug(f'Start converting "{filename.name}".')
ruleset_custom = ruleset.custom_convert(filename)
ruleset_custom = ruleset.load(filename)
ruleset.batch_dump(ruleset_custom, const.TARGETS, const.PATH_DIST, filename.stem)
logger.debug(f"Converted {len(ruleset_custom)} rules.")

# There's no personal classical type ruleset. So no logic about that.
list_file_personal = Path.iterdir(const.PATH_SOURCE_CUSTOM/"personal")
for filename in list_file_personal:
logger.debug(f'Start converting "{filename.name}".')
ruleset_personal = ruleset.custom_convert(filename)
ruleset_personal = ruleset.load(filename)
ruleset.batch_dump(ruleset_personal, ["text", "text-plus", "yaml", "surge-compatible", "clash-compatible"],
const.PATH_DIST/"personal", filename.stem)
ruleset.dump(ruleset_personal, "geosite", const.PATH_DIST/"geosite", ("personal-" + filename.stem))
Expand Down

0 comments on commit 8fb4a06

Please sign in to comment.