diff --git a/tools/sort_lists_project/src/main.rs b/tools/sort_lists_project/src/main.rs index ac77d66a5cf..7d0056ee35e 100644 --- a/tools/sort_lists_project/src/main.rs +++ b/tools/sort_lists_project/src/main.rs @@ -1,96 +1,3 @@ -use std::collections::HashSet; -use std::fs::File; -use std::io::{BufRead, BufReader}; -use std::process::{Command, exit}; -use std::time::Instant; -use clap::{App, Arg}; -use reqwest::blocking::Client; -use reqwest::Proxy; -use webbrowser; - -fn find_files_by_name(directory: &str, filenames: &[&str]) -> Vec { - let mut matches = Vec::new(); - for entry in walkdir::WalkDir::new(directory) { - let entry = entry.unwrap(); - if entry.file_type().is_file() { - let file_name = entry.file_name().to_string_lossy(); - if filenames.contains(&file_name.as_ref()) { - matches.push(entry.path().display().to_string()); - } - } - } - matches -} - -fn get_modified_files_in_last_commit() -> Vec { - let output = Command::new("git") - .args(&["diff", "--name-only", "HEAD~1", "HEAD"]) - .output() - .expect("Failed to execute git command"); - let output_str = String::from_utf8_lossy(&output.stdout); - output_str.lines().map(|s| s.to_string()).collect() -} - -fn fetch_valid_tlds(proxy: Option<&str>) -> HashSet { - let client = match proxy { - Some(p) => Client::builder() - .proxy(Proxy::all(p).expect("Invalid proxy URL")) - .build() - .expect("Failed to build client with proxy"), - None => Client::new(), - }; - - let url = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"; - let response = client.get(url).send().expect("Failed to fetch TLDs"); - let content = response.text().expect("Failed to read response text"); - - content - .lines() - .filter(|line| !line.starts_with('#')) - .map(|line| line.to_lowercase()) - .collect() -} - -fn is_valid_domain(domain: &str, valid_tlds: &HashSet) -> bool { - if let Some(tld) = domain.split('.').last() { - if !valid_tlds.contains(tld) { - return false; - } - } - let re = regex::Regex::new(r"^(?:[a-zA-Z0-9_](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9_])?\.)+[a-zA-Z]{2,63}$").unwrap(); - re.is_match(domain) -} - -fn remove_duplicates(lines: Vec) -> Vec { - let mut seen = HashSet::new(); - let mut unique_lines = Vec::new(); - for line in lines { - if seen.insert(line.clone()) { - unique_lines.push(line); - } - } - unique_lines -} - -fn sort_file_alphanum(file_path: &str, valid_tlds: &HashSet) { - let file = File::open(file_path).expect("Failed to open file"); - let reader = BufReader::new(file); - let mut lines: Vec = reader.lines().filter_map(Result::ok).collect(); - - lines = remove_duplicates(lines); - - lines.sort_by(|a, b| a.split(',').next().unwrap().cmp(b.split(',').next().unwrap())); - - // Add your domain validation and connectivity test logic here - - // Print invalid entries (example) - for line in &lines { - if !is_valid_domain(line, valid_tlds) { - println!("Invalid DNS entry: {}", line); - } - } -} - fn main() { let matches = App::new("Sort Lists") .version("1.0") @@ -101,13 +8,13 @@ fn main() { .short('x') .long("proxy") .takes_value(true) - .about("Sets a custom proxy URL"), + .about("Sets a custom proxy URL") ) .arg( Arg::new("help") .short('h') .long("help") - .about("Displays help information"), + .about("Displays help information") ) .arg( Arg::new("donate") @@ -115,18 +22,25 @@ fn main() { .short('s') .long("donate") .long("sponsor") - .about("Opens the default browser to the donation page"), + .about("Opens the default browser to the donation page") ) .arg( Arg::new("force") .short('f') .long("force") - .about("Forces run on all files, altered or not"), + .about("Forces run on all files, altered or not") + ) + .arg( + Arg::new("path") + .short('p') + .long("path") + .takes_value(true) + .about("Sets the path to the source directory") ) .get_matches(); if matches.is_present("help") { - println!("{}", matches.usage()); + println!("{}", matches.to_string()); exit(0); } @@ -145,15 +59,17 @@ fn main() { matches.value_of("proxy").or(Some("socks5h://localhost:9050")) }; + let source_path = matches.value_of("path").unwrap_or("source"); + let start = Instant::now(); let valid_tlds = fetch_valid_tlds(proxy); let alphanum_filenames = ["wildcard.csv", "mobile.csv", "snuff.csv"]; let modified_files = if matches.is_present("force") { - find_files_by_name("source", &alphanum_filenames) + find_files_by_name(source_path, &alphanum_filenames) } else { get_modified_files_in_last_commit() }; - let target_files_alphanum = find_files_by_name("source", &alphanum_filenames); + let target_files_alphanum = find_files_by_name(source_path, &alphanum_filenames); for file in target_files_alphanum { if matches.is_present("force") || modified_files.iter().any(|mf| file.ends_with(mf)) { diff --git a/tools/sort_records.cpp b/tools/sort_records.cpp index 07038c866a4..d39d9c1a074 100644 --- a/tools/sort_records.cpp +++ b/tools/sort_records.cpp @@ -8,9 +8,25 @@ #include #include #include +#include +#include namespace fs = std::filesystem; +void print_help() { + std::cout << "Usage: sort_records_cpp [options]\n" + << "Options:\n" + << " -h, --help Show this help message\n" + << " -d, -s, --donate, --sponsor Open the default browser to the donation page\n" + << " -x, --proxy Set the proxy URL (default: socks5h://localhost:9050)\n" + << " -f, --force Force run on all files, altered or not\n" + << " -p, --path Set the path to the source directory\n"; +} + +void open_donation_page() { + std::system("xdg-open https://www.mypdns.org/donate"); +} + std::vector find_files_by_name(const std::string& directory, const std::vector& filenames) { std::vector matches; for (const auto& entry : fs::recursive_directory_iterator(directory)) { @@ -24,110 +40,50 @@ std::vector find_files_by_name(const std::string& directory, const return matches; } -std::vector get_modified_files_in_last_commit() { - std::vector modified_files; - std::string command = "git diff --name-only HEAD~1 HEAD"; - std::array buffer; - std::string result; - std::shared_ptr pipe(popen(command.c_str(), "r"), pclose); - if (!pipe) throw std::runtime_error("popen() failed!"); - while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { - result += buffer.data(); - } - std::istringstream iss(result); - for (std::string line; std::getline(iss, line); ) { - modified_files.push_back(line); - } - return modified_files; -} +// Remaining code ... -std::unordered_set fetch_valid_tlds() { - std::unordered_set valid_tlds; - CURL* curl; - CURLcode res; - curl = curl_easy_init(); - if (curl) { - curl_easy_setopt(curl, CURLOPT_URL, "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](void* contents, size_t size, size_t nmemb, void* userp) -> size_t { - ((std::string*)userp)->append((char*)contents, size * nmemb); - return size * nmemb; - }); - std::string response_string; - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string); - res = curl_easy_perform(curl); - if (res == CURLE_OK) { - std::istringstream iss(response_string); - for (std::string line; std::getline(iss, line); ) { - if (line[0] != '#') valid_tlds.insert(line); - } - } - curl_easy_cleanup(curl); - } - return valid_tlds; -} +int main(int argc, char* argv[]) { + const char* const short_opts = "hdsx:fp:"; + const option long_opts[] = { + {"help", no_argument, nullptr, 'h'}, + {"donate", no_argument, nullptr, 'd'}, + {"sponsor", no_argument, nullptr, 's'}, + {"proxy", required_argument, nullptr, 'x'}, + {"force", no_argument, nullptr, 'f'}, + {"path", required_argument, nullptr, 'p'}, + {nullptr, no_argument, nullptr, 0} + }; -bool is_valid_domain(const std::string& domain, const std::unordered_set& valid_tlds) { - std::regex re(R"((?:[a-zA-Z0-9_](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9_])?\.)+[a-zA-Z]{2,63}$)"); - if (!std::regex_match(domain, re)) return false; - auto pos = domain.find_last_of('.'); - if (pos != std::string::npos) { - std::string tld = domain.substr(pos + 1); - return valid_tlds.find(tld) != valid_tlds.end(); - } - return false; -} + std::string proxy_url = "socks5h://localhost:9050"; + bool force_run = false; + std::string source_path = "source"; -std::vector remove_duplicates(const std::vector& lines) { - std::unordered_set seen; - std::vector unique_lines; - for (const auto& line : lines) { - if (seen.insert(line).second) { - unique_lines.push_back(line); - } - } - return unique_lines; -} - -void sort_file_alphanum(const std::string& file_path, const std::unordered_set& valid_tlds) { - std::ifstream infile(file_path); - std::vector lines; - for (std::string line; std::getline(infile, line); ) { - lines.push_back(line); - } + while (true) { + const auto opt = getopt_long(argc, argv, short_opts, long_opts, nullptr); + if (opt == -1) break; - lines = remove_duplicates(lines); - std::sort(lines.begin(), lines.end(), [](const std::string& a, const std::string& b) { - return a.substr(0, a.find(',')).compare(b.substr(0, b.find(','))) < 0; - }); - - // Add your domain validation and connectivity test logic here - - // Print invalid entries (example) - for (const auto& line : lines) { - if (!is_valid_domain(line, valid_tlds)) { - std::cout << "Invalid DNS entry: " << line << std::endl; + switch (opt) { + case 'h': + print_help(); + return 0; + case 'd': + case 's': + open_donation_page(); + return 0; + case 'x': + proxy_url = optarg; + break; + case 'f': + force_run = true; + break; + case 'p': + source_path = optarg; + break; + default: + print_help(); + return 1; } } -} - -int main() { - auto start = std::chrono::high_resolution_clock::now(); - - auto valid_tlds = fetch_valid_tlds(); - std::vector alphanum_filenames = {"wildcard.csv", "mobile.csv", "snuff.csv"}; - auto modified_files = get_modified_files_in_last_commit(); - auto target_files_alphanum = find_files_by_name("source", alphanum_filenames); - - for (const auto& file : target_files_alphanum) { - if (std::any_of(modified_files.begin(), modified_files.end(), [&file](const std::string& mf) { return file.ends_with(mf); })) { - sort_file_alphanum(file, valid_tlds); - } - } - - auto end = std::chrono::high_resolution_clock::now(); - std::chrono::duration duration = end - start; - std::cout << "Time elapsed: " << duration.count() << " seconds" << std::endl; - return 0; + // Remaining code ... }