Skip to content

Commit

Permalink
Testing rust, c++ vs python3.12 BUILD Bob Build (19)
Browse files Browse the repository at this point in the history
  • Loading branch information
@spirillen
spirillen committed Jan 18, 2025
1 parent 8503968 commit 60d61a9
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 199 deletions.
116 changes: 16 additions & 100 deletions tools/sort_lists_project/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,96 +1,3 @@
use std::collections::HashSet;
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::process::{Command, exit};
use std::time::Instant;
use clap::{App, Arg};
use reqwest::blocking::Client;
use reqwest::Proxy;
use webbrowser;

fn find_files_by_name(directory: &str, filenames: &[&str]) -> Vec<String> {
let mut matches = Vec::new();
for entry in walkdir::WalkDir::new(directory) {
let entry = entry.unwrap();
if entry.file_type().is_file() {
let file_name = entry.file_name().to_string_lossy();
if filenames.contains(&file_name.as_ref()) {
matches.push(entry.path().display().to_string());
}
}
}
matches
}

fn get_modified_files_in_last_commit() -> Vec<String> {
let output = Command::new("git")
.args(&["diff", "--name-only", "HEAD~1", "HEAD"])
.output()
.expect("Failed to execute git command");
let output_str = String::from_utf8_lossy(&output.stdout);
output_str.lines().map(|s| s.to_string()).collect()
}

fn fetch_valid_tlds(proxy: Option<&str>) -> HashSet<String> {
let client = match proxy {
Some(p) => Client::builder()
.proxy(Proxy::all(p).expect("Invalid proxy URL"))
.build()
.expect("Failed to build client with proxy"),
None => Client::new(),
};

let url = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt";
let response = client.get(url).send().expect("Failed to fetch TLDs");
let content = response.text().expect("Failed to read response text");

content
.lines()
.filter(|line| !line.starts_with('#'))
.map(|line| line.to_lowercase())
.collect()
}

fn is_valid_domain(domain: &str, valid_tlds: &HashSet<String>) -> bool {
if let Some(tld) = domain.split('.').last() {
if !valid_tlds.contains(tld) {
return false;
}
}
let re = regex::Regex::new(r"^(?:[a-zA-Z0-9_](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9_])?\.)+[a-zA-Z]{2,63}$").unwrap();
re.is_match(domain)
}

fn remove_duplicates(lines: Vec<String>) -> Vec<String> {
let mut seen = HashSet::new();
let mut unique_lines = Vec::new();
for line in lines {
if seen.insert(line.clone()) {
unique_lines.push(line);
}
}
unique_lines
}

fn sort_file_alphanum(file_path: &str, valid_tlds: &HashSet<String>) {
let file = File::open(file_path).expect("Failed to open file");
let reader = BufReader::new(file);
let mut lines: Vec<String> = reader.lines().filter_map(Result::ok).collect();

lines = remove_duplicates(lines);

lines.sort_by(|a, b| a.split(',').next().unwrap().cmp(b.split(',').next().unwrap()));

// Add your domain validation and connectivity test logic here

// Print invalid entries (example)
for line in &lines {
if !is_valid_domain(line, valid_tlds) {
println!("Invalid DNS entry: {}", line);
}
}
}

fn main() {
let matches = App::new("Sort Lists")
.version("1.0")
Expand All @@ -101,32 +8,39 @@ fn main() {
.short('x')
.long("proxy")
.takes_value(true)
.about("Sets a custom proxy URL"),
.about("Sets a custom proxy URL")
)
.arg(
Arg::new("help")
.short('h')
.long("help")
.about("Displays help information"),
.about("Displays help information")
)
.arg(
Arg::new("donate")
.short('d')
.short('s')
.long("donate")
.long("sponsor")
.about("Opens the default browser to the donation page"),
.about("Opens the default browser to the donation page")
)
.arg(
Arg::new("force")
.short('f')
.long("force")
.about("Forces run on all files, altered or not"),
.about("Forces run on all files, altered or not")
)
.arg(
Arg::new("path")
.short('p')
.long("path")
.takes_value(true)
.about("Sets the path to the source directory")
)
.get_matches();

if matches.is_present("help") {
println!("{}", matches.usage());
println!("{}", matches.to_string());
exit(0);
}

Expand All @@ -145,15 +59,17 @@ fn main() {
matches.value_of("proxy").or(Some("socks5h://localhost:9050"))
};

let source_path = matches.value_of("path").unwrap_or("source");

let start = Instant::now();
let valid_tlds = fetch_valid_tlds(proxy);
let alphanum_filenames = ["wildcard.csv", "mobile.csv", "snuff.csv"];
let modified_files = if matches.is_present("force") {
find_files_by_name("source", &alphanum_filenames)
find_files_by_name(source_path, &alphanum_filenames)
} else {
get_modified_files_in_last_commit()
};
let target_files_alphanum = find_files_by_name("source", &alphanum_filenames);
let target_files_alphanum = find_files_by_name(source_path, &alphanum_filenames);

for file in target_files_alphanum {
if matches.is_present("force") || modified_files.iter().any(|mf| file.ends_with(mf)) {
Expand Down
154 changes: 55 additions & 99 deletions tools/sort_records.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,25 @@
#include <filesystem>
#include <chrono>
#include <curl/curl.h>
#include <cstdlib>
#include <getopt.h>

namespace fs = std::filesystem;

void print_help() {
std::cout << "Usage: sort_records_cpp [options]\n"
<< "Options:\n"
<< " -h, --help Show this help message\n"
<< " -d, -s, --donate, --sponsor Open the default browser to the donation page\n"
<< " -x, --proxy Set the proxy URL (default: socks5h://localhost:9050)\n"
<< " -f, --force Force run on all files, altered or not\n"
<< " -p, --path Set the path to the source directory\n";
}

void open_donation_page() {
std::system("xdg-open https://www.mypdns.org/donate");
}

std::vector<std::string> find_files_by_name(const std::string& directory, const std::vector<std::string>& filenames) {
std::vector<std::string> matches;
for (const auto& entry : fs::recursive_directory_iterator(directory)) {
Expand All @@ -24,110 +40,50 @@ std::vector<std::string> find_files_by_name(const std::string& directory, const
return matches;
}

std::vector<std::string> get_modified_files_in_last_commit() {
std::vector<std::string> modified_files;
std::string command = "git diff --name-only HEAD~1 HEAD";
std::array<char, 128> buffer;
std::string result;
std::shared_ptr<FILE> pipe(popen(command.c_str(), "r"), pclose);
if (!pipe) throw std::runtime_error("popen() failed!");
while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) {
result += buffer.data();
}
std::istringstream iss(result);
for (std::string line; std::getline(iss, line); ) {
modified_files.push_back(line);
}
return modified_files;
}
// Remaining code ...

std::unordered_set<std::string> fetch_valid_tlds() {
std::unordered_set<std::string> valid_tlds;
CURL* curl;
CURLcode res;
curl = curl_easy_init();
if (curl) {
curl_easy_setopt(curl, CURLOPT_URL, "https://data.iana.org/TLD/tlds-alpha-by-domain.txt");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, +[](void* contents, size_t size, size_t nmemb, void* userp) -> size_t {
((std::string*)userp)->append((char*)contents, size * nmemb);
return size * nmemb;
});
std::string response_string;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response_string);
res = curl_easy_perform(curl);
if (res == CURLE_OK) {
std::istringstream iss(response_string);
for (std::string line; std::getline(iss, line); ) {
if (line[0] != '#') valid_tlds.insert(line);
}
}
curl_easy_cleanup(curl);
}
return valid_tlds;
}
int main(int argc, char* argv[]) {
const char* const short_opts = "hdsx:fp:";
const option long_opts[] = {
{"help", no_argument, nullptr, 'h'},
{"donate", no_argument, nullptr, 'd'},
{"sponsor", no_argument, nullptr, 's'},
{"proxy", required_argument, nullptr, 'x'},
{"force", no_argument, nullptr, 'f'},
{"path", required_argument, nullptr, 'p'},
{nullptr, no_argument, nullptr, 0}
};

bool is_valid_domain(const std::string& domain, const std::unordered_set<std::string>& valid_tlds) {
std::regex re(R"((?:[a-zA-Z0-9_](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9_])?\.)+[a-zA-Z]{2,63}$)");
if (!std::regex_match(domain, re)) return false;
auto pos = domain.find_last_of('.');
if (pos != std::string::npos) {
std::string tld = domain.substr(pos + 1);
return valid_tlds.find(tld) != valid_tlds.end();
}
return false;
}
std::string proxy_url = "socks5h://localhost:9050";
bool force_run = false;
std::string source_path = "source";

std::vector<std::string> remove_duplicates(const std::vector<std::string>& lines) {
std::unordered_set<std::string> seen;
std::vector<std::string> unique_lines;
for (const auto& line : lines) {
if (seen.insert(line).second) {
unique_lines.push_back(line);
}
}
return unique_lines;
}

void sort_file_alphanum(const std::string& file_path, const std::unordered_set<std::string>& valid_tlds) {
std::ifstream infile(file_path);
std::vector<std::string> lines;
for (std::string line; std::getline(infile, line); ) {
lines.push_back(line);
}
while (true) {
const auto opt = getopt_long(argc, argv, short_opts, long_opts, nullptr);
if (opt == -1) break;

lines = remove_duplicates(lines);
std::sort(lines.begin(), lines.end(), [](const std::string& a, const std::string& b) {
return a.substr(0, a.find(',')).compare(b.substr(0, b.find(','))) < 0;
});

// Add your domain validation and connectivity test logic here

// Print invalid entries (example)
for (const auto& line : lines) {
if (!is_valid_domain(line, valid_tlds)) {
std::cout << "Invalid DNS entry: " << line << std::endl;
switch (opt) {
case 'h':
print_help();
return 0;
case 'd':
case 's':
open_donation_page();
return 0;
case 'x':
proxy_url = optarg;
break;
case 'f':
force_run = true;
break;
case 'p':
source_path = optarg;
break;
default:
print_help();
return 1;
}
}
}

int main() {
auto start = std::chrono::high_resolution_clock::now();

auto valid_tlds = fetch_valid_tlds();
std::vector<std::string> alphanum_filenames = {"wildcard.csv", "mobile.csv", "snuff.csv"};
auto modified_files = get_modified_files_in_last_commit();
auto target_files_alphanum = find_files_by_name("source", alphanum_filenames);

for (const auto& file : target_files_alphanum) {
if (std::any_of(modified_files.begin(), modified_files.end(), [&file](const std::string& mf) { return file.ends_with(mf); })) {
sort_file_alphanum(file, valid_tlds);
}
}

auto end = std::chrono::high_resolution_clock::now();
std::chrono::duration<double> duration = end - start;
std::cout << "Time elapsed: " << duration.count() << " seconds" << std::endl;

return 0;
// Remaining code ...
}

0 comments on commit 60d61a9

Please sign in to comment.