Taevas/Website-Finder
Taevas/Website-Finder
0
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make use of defaults.json (except C# for now)

Browse source
This commit is contained in:
Taevas 2022-12-12 21:03:40 +01:00
parent a8bda4704a
commit 8656fd134f
6 changed files with 102 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,4 +1,4 @@
*.json *_report_*.json
## Ignore Visual Studio temporary files, build results, and ## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons. ## files generated by popular Visual Studio add-ons.

33
Crystal/index.cr
View file

@ -8,7 +8,7 @@ def main_loop(times, domains, protocols, log, min, max, second, report_file)
json.array do json.array do
i = 0 i = 0
while i < times while i < times
url = url_generator(domains, protocols, min.as(UInt8), max.as(UInt8), second) url = url_generator(domains, protocols, min, max, second)
puts "#{url} (#{i + 1}/#{times})" if log puts "#{url} (#{i + 1}/#{times})" if log
client = HTTP::Client.new(URI.parse url) client = HTTP::Client.new(URI.parse url)
client.connect_timeout = 40.seconds client.connect_timeout = 40.seconds
@ -46,7 +46,7 @@ end
def url_generator(domains, protocols, min, max, second) def url_generator(domains, protocols, min, max, second)
result = String.build do |str| result = String.build do |str|
str << protocols[Random.rand(protocols.size)] + "://" str << "#{protocols[Random.rand(protocols.size)]}://"
url_length = Random.rand(min..max) url_length = Random.rand(min..max)
characters = "abcdefghijklmnopqrstuvwxyz0123456789" characters = "abcdefghijklmnopqrstuvwxyz0123456789"
i = 0 i = 0
@ -54,19 +54,20 @@ def url_generator(domains, protocols, min, max, second)
str << characters[Random.rand(characters.size - 1)] str << characters[Random.rand(characters.size - 1)]
i += 1 i += 1
end end
str << domains[Random.rand(domains.size)] if Random.rand(1..100) <= second str << ".#{domains[Random.rand(domains.size)]}" if Random.rand(1..100) <= second
str << domains[Random.rand(domains.size)] str << ".#{domains[Random.rand(domains.size)]}"
end end
result result
end end
times = UInt32.new "3000" defaults = JSON.parse(File.read("../defaults.json"))
protocols = ["http"] times = defaults["times"].as_i
domains = [".co", ".com", ".net", ".edu", ".gov", ".cn", ".org", ".cc", ".us", ".mil", ".ac", ".it", ".de"] protocols = defaults["protocols"].as_a
second = UInt8.new "1" domains = defaults["domains"].as_a
log = false second = defaults["second"].as_i
min = UInt8.new "2" log = defaults["log"].as_bool
max = UInt8.new "50" min = defaults["min"].as_i
max = defaults["max"].as_i
OptionParser.parse do |parser| OptionParser.parse do |parser|
parser.banner = "Website-Finder" parser.banner = "Website-Finder"
@ -74,19 +75,19 @@ OptionParser.parse do |parser|
puts parser puts parser
exit exit
end end
parser.on("-t TIMES", "--times=TIMES", "Number of requests / DEFAULT: #{times}") {|p_times| times = p_times.to_u32} parser.on("-t TIMES", "--times=TIMES", "Number of requests / DEFAULT: #{times}") {|p_times| times = p_times.to_i}
parser.on("-d DOMAINS", "--domains=DOMAINS", "Domains used in URLS, like: .com,.net,.gov / DEFAULT: #{domains}") {|p_domains| domains = p_domains.split(",")} parser.on("-d DOMAINS", "--domains=DOMAINS", "Domains used in URLS, like: .com,.net,.gov / DEFAULT: #{domains}") {|p_domains| domains = p_domains.split(",")}
parser.on("-p protocols", "--protocols=PROTOCOLS", "You may choose between: http | https | http,https / DEFAULT: #{protocols}") {|p_protocols| protocols = p_protocols.split(",")} parser.on("-p protocols", "--protocols=PROTOCOLS", "You may choose between: http | https | http,https / DEFAULT: #{protocols}") {|p_protocols| protocols = p_protocols.split(",")}
parser.on("-l", "--log", "Log all requests / DEFAULT: #{log}") {log = true} parser.on("-l", "--log", "Log all requests / DEFAULT: #{log}") {log = true}
parser.on("", "--min=LENGTH", "Minimum length of URLs / DEFAULT: #{min}") {|p_length| min = p_length.to_u8} parser.on("", "--min=LENGTH", "Minimum length of URLs / DEFAULT: #{min}") {|p_length| min = p_length.to_i}
parser.on("", "--max=LENGTH", "Maximum length of URLs / DEFAULT: #{max}") {|p_length| max = p_length.to_u8} parser.on("", "--max=LENGTH", "Maximum length of URLs / DEFAULT: #{max}") {|p_length| max = p_length.to_i}
parser.on("-s SECOND", "--second=SECOND", "Likelihood of a URL featuring a second-level domain / DEFAULT: #{second}") {|p_second| second = p_second.to_u8} parser.on("-s SECOND", "--second=SECOND", "Likelihood of a URL featuring a second-level domain / DEFAULT: #{second}") {|p_second| second = p_second.to_i}
end end
date = Time.local date = Time.local
puts "\nI am going to look for websites through #{times} random URLs (min length #{min} and max length #{max} with the following domains: #{domains}" puts "\nI am going to look for websites through #{times} random URLs (min length #{min} and max length #{max} with the following domains: #{domains}"
puts "These URLs will use the protocols #{protocols}" puts "These URLs will use the protocols #{protocols}"
puts "and each of those URLs have #{second} in a 100 chance to have a second level domain." puts "and each of those URLs have #{second} in a 100 chance to have a second level domain"
puts "Started at #{date.hour}h#{date.minute}m\n" puts "Started at #{date.hour}h#{date.minute}m\n"
report_file = "CR_report_#{date.day}#{date.hour}#{date.minute}.json" report_file = "CR_report_#{date.day}#{date.hour}#{date.minute}.json"

40
Node.js/index.js
View file

@ -5,16 +5,16 @@ const fs = require('fs')
async function main_loop() { async function main_loop() {
json_object = [] json_object = []
for (let i = 0; i < times; i++) { for (let i = 0; i < times; i++) {
const url = await url_generator(i, times) const url = url_generator(i, times)
try { try {
const response = await fetch(url) const response = await fetch(url)
console.log(`${url} exists!`) console.log(`${url} exists!`)
json_object.push(`{"website_url":"${url}","response_type":"SUCCESS","response_code":"${String(response.statusCode)}","response_details":"${String(response.statusMessage)}"}`) json_object.push(`{"website_url":"${url}","response_type":"SUCCESS","response_code":"${response.statusCode}","response_details":"${response.statusMessage}"}`)
} }
catch(e) { catch(e) {
if (e.code != 'ENOTFOUND') { if (e.code != 'ENOTFOUND') {
console.log(`${url} exists!`) console.log(`${url} exists!`)
json_object.push(`{"website_url":"${url}","response_type":"ERROR","response_code":"${String(e.code)}","response_details":"${String(e.syscall)}"}`) json_object.push(`{"website_url":"${url}","response_type":"ERROR","response_code":"${e.code}","response_details":"${e.syscall}"}`)
} }
} }
} }
@ -27,21 +27,18 @@ function url_generator(num_url, times) {
const characters = "abcdefghijklmnopqrstuvwxyz0123456789" const characters = "abcdefghijklmnopqrstuvwxyz0123456789"
const url_length = Math.floor(Math.random() * (maxi - mini) + mini) const url_length = Math.floor(Math.random() * (maxi - mini) + mini)
for (let i = 0; i < url_length; i++) {result += characters.charAt(Math.floor(Math.random() * characters.length))} for (let i = 0; i < url_length; i++) {result += characters.charAt(Math.floor(Math.random() * characters.length))}
result += domains[Math.floor(Math.random() * domains.length)] result += `.${domains[Math.floor(Math.random() * domains.length)]}`
if (Math.floor(Math.random() * (100 - 1) + 1) <= second) result += domains[Math.floor(Math.random() * domains.length)] if (Math.floor(Math.random() * (100 - 1) + 1) <= second) result += `.${domains[Math.floor(Math.random() * domains.length)]}`
if (log) console.log(`${result} (${num_url + 1}/${times})`) if (log) console.log(`${result} (${num_url + 1}/${times})`)
return result return result
} }
function fetch(url, options = {}) { function fetch(url, options = {}) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!url) return reject(new Error('URL was not provided')) // Cannot happen; exists just for the sake of it
const {body, method = 'GET', ...restOptions} = options
const client = url.startsWith('https') ? https : http const client = url.startsWith('https') ? https : http
const request = client.request(url, {method, ...restOptions}, (res) => { const request = client.request(url, {method: "GET"}, (res) => {
res.setEncoding('utf8') res.setEncoding('utf8')
res.on('data', (chunk) => {}) // Do nothing, it must handle receiving data but we do not need the received data res.on('data', () => {}) // Do nothing, deleting this line actually makes the software exit upon finding a website (wtf)
res.on('end', () => {resolve({statusCode: res.statusCode, statusMessage: res.statusMessage})}) res.on('end', () => {resolve({statusCode: res.statusCode, statusMessage: res.statusMessage})})
}) })
request.on('error', (err) => {reject(err)}) request.on('error', (err) => {reject(err)})
@ -49,22 +46,23 @@ function fetch(url, options = {}) {
}) })
} }
const times = process.argv.indexOf('-t') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-t') + 1])) : 3000 const defaults = require("../defaults.json")
const protocols = process.argv.indexOf('-p') > -1 ? process.argv[process.argv.indexOf('-p') + 1].split(',') : ['http'] const times = process.argv.indexOf('-t') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-t') + 1])) : defaults.times
const domains = process.argv.indexOf('-d') > -1 ? process.argv[process.argv.indexOf('-d') + 1].split(',') : ['.co', '.com', '.net', '.edu', '.gov', '.cn', '.org', '.cc', '.us', '.mil', '.ac', '.it', '.de'] const protocols = process.argv.indexOf('-p') > -1 ? process.argv[process.argv.indexOf('-p') + 1].split(',') : defaults.protocols
const second = process.argv.indexOf('-s') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-s') + 1])) : 1 const domains = process.argv.indexOf('-d') > -1 ? process.argv[process.argv.indexOf('-d') + 1].split(',') : defaults.domains
const log = process.argv.indexOf('-l') > -1 const second = process.argv.indexOf('-s') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-s') + 1])) : defaults.second
const mini = process.argv.indexOf('-min') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-min') + 1])) : 2 const log = process.argv.indexOf('-l') > -1 ? true : defaults.log
const maxi = process.argv.indexOf('-max') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-max') + 1])) : 50 const mini = process.argv.indexOf('-min') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-min') + 1])) : defaults.min
const maxi = process.argv.indexOf('-max') > -1 ? Math.round(Number(process.argv[process.argv.indexOf('-max') + 1])) : defaults.max
const report_file = "JS_report_" + String(new Date().getUTCDate()) + String(new Date().getHours()) + String(new Date().getMinutes()) + ".json"
const date = new Date()
process.stdout.write(`\nI am going to look for websites through ${times} random URLs (min length ${mini} and max length ${maxi}) with the following domains: `) process.stdout.write(`\nI am going to look for websites through ${times} random URLs (min length ${mini} and max length ${maxi}) with the following domains: `)
console.log(domains) console.log(domains)
process.stdout.write("These URLs will use the protocols ") process.stdout.write("These URLs will use the protocols ")
console.log(protocols) console.log(protocols)
console.log(`and each of them have ${second} in a 100 chance to have a second level domain.`) console.log(`and each of them have ${second} in a 100 chance to have a second level domain`)
console.log('Started at ' + String(new Date().getHours()) + 'h' + String(new Date().getMinutes()) + 'm\n') console.log("Started at %dh%dm\n", date.getHours(), date.getMinutes())
const report_file = "JS_report_" + String(date.getUTCDate()) + String(date.getHours()) + String(date.getMinutes()) + ".json"
fs.open(report_file, "w", function(err) {if (err) throw err}) fs.open(report_file, "w", function(err) {if (err) throw err})
main_loop() main_loop()

38
Python/index.py
View file

@ -2,6 +2,7 @@ import sys
import random import random
import datetime import datetime
import requests import requests
import json
from requests.packages.urllib3.exceptions import InsecureRequestWarning from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning) requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
@ -16,13 +17,14 @@ def main_loop():
json_object.append('{"website_url":"' + url + '","response_type":"SUCCESS","response_code":"' + str(response.status_code) + '","response_details":"' + str(response.reason) + '"}') json_object.append('{"website_url":"' + url + '","response_type":"SUCCESS","response_code":"' + str(response.status_code) + '","response_details":"' + str(response.reason) + '"}')
except Exception as e: # Exception should always be ConnectionError (**usually** bad) or ReadTimeout (good) except Exception as e: # Exception should always be ConnectionError (**usually** bad) or ReadTimeout (good)
# Exception handling seems to be a pain because most errors return ConnectionError, so ConnectionError in itself can mean the website exists OR the website does NOT exist # Exception handling seems to be a pain because most errors return ConnectionError, so ConnectionError in itself can mean the website exists OR the website does NOT exist
if "not known" not in str(e).lower() and "no address" not in str(e).lower(): err = str(e)
if "not known" not in err.lower() and "no address" not in err.lower():
print(url + " exists!") print(url + " exists!")
err_code = str(e)[str(e).index("[") + 1:str(e).index("]")] if "[" in str(e) and "]" in str(e) else "NO CODE FOUND" err_code = err[err.index("[")+1 : err.index("]")] if "[" in err and "]" in err else "NO CODE FOUND"
json_object.append('{"website_url":"' + url + '","response_type":"ERROR","response_code":"' + err_code + '","response_details":"' + str(e).replace("\\", "").replace('"', "") + '"}') json_object.append('{"website_url":"' + url + '","response_type":"ERROR","response_code":"' + err_code + '","response_details":"' + err.replace("\\", "").replace('"', "") + '"}')
f.write(str(json_object).replace("'", "").replace("\\", "")) report_file.write(str(json_object).replace("'", "").replace("\\", ""))
f.close() report_file.close()
print("Finished at " + str(datetime.datetime.now().time())[0:5].replace(":", "h") + "m") print("Finished at " + str(datetime.datetime.now().time())[0:5].replace(":", "h") + "m")
def url_generator(num_url, times): def url_generator(num_url, times):
@ -30,23 +32,25 @@ def url_generator(num_url, times):
characters = "abcdefghijklmnopqrstuvwxyz0123456789" characters = "abcdefghijklmnopqrstuvwxyz0123456789"
url_length = random.randint(min, max) url_length = random.randint(min, max)
result += ''.join(random.choice(characters) for i in range(url_length)) result += ''.join(random.choice(characters) for i in range(url_length))
result += domains[random.randint(0, len(domains) - 1)] result += f".{domains[random.randint(0, len(domains) - 1)]}"
if random.randint(1, 100) <= second: result += domains[random.randint(0, len(domains) - 1)] if random.randint(1, 100) <= second: result += ".%s"%(domains[random.randint(0, len(domains) - 1)])
if log: print(result + " (" + str(num_url + 1) + "/" + str(times) + ")") if log: print(result + " (" + str(num_url + 1) + "/" + str(times) + ")")
return result return result
times = int(sys.argv[sys.argv.index('-t') + 1]) if '-t' in sys.argv else 3000 defaults = json.load(open("../defaults.json", "rb"))
protocols = sys.argv[sys.argv.index('-p') + 1].split(",") if '-p' in sys.argv else ['http'] times = int(sys.argv[sys.argv.index('-t') + 1]) if '-t' in sys.argv else defaults["times"]
domains = sys.argv[sys.argv.index('-d') + 1].split(",") if '-d' in sys.argv else ['.co', '.com', '.net', '.edu', '.gov', '.cn', '.org', '.cc', '.us', '.mil', '.ac', '.it', '.de'] protocols = sys.argv[sys.argv.index('-p') + 1].split(",") if '-p' in sys.argv else defaults["protocols"]
second = int(sys.argv[sys.argv.index('-s') + 1]) if '-s' in sys.argv else 1 domains = sys.argv[sys.argv.index('-d') + 1].split(",") if '-d' in sys.argv else defaults["domains"]
log = '-l' in sys.argv second = int(sys.argv[sys.argv.index('-s') + 1]) if '-s' in sys.argv else defaults["second"]
log = True if '-l' in sys.argv else defaults["log"]
# lmao what if we literally get rid of two built-in functions # lmao what if we literally get rid of two built-in functions
min = int(sys.argv[sys.argv.index('-min') + 1]) if '-min' in sys.argv else 2 min = int(sys.argv[sys.argv.index('-min') + 1]) if '-min' in sys.argv else defaults["min"]
max = int(sys.argv[sys.argv.index('-max') + 1]) if '-max' in sys.argv else 50 # Python cannot look for URLs longer than 50ish, so be careful! max = int(sys.argv[sys.argv.index('-max') + 1]) if '-max' in sys.argv else defaults["max"] # Avoid >50
date = datetime.datetime.now()
print("\nI am going to look for websites through " + str(times) + " random URLs (min length " + str(min) + " and max length " + str(max) + ") with the following domains: " + str(domains)) print("\nI am going to look for websites through " + str(times) + " random URLs (min length " + str(min) + " and max length " + str(max) + ") with the following domains: " + str(domains))
print("These URLs will use the protocols " + str(protocols) + " and each of those URLs have " + str(second) + " in 100 chance to have a second level domain.") print("These URLs will use the protocols " + str(protocols) + " and each of those URLs have " + str(second) + " in 100 chance to have a second level domain")
print("Started at " + str(datetime.datetime.now().time())[0:5].replace(":", "h") + "m\n") print("Started at " + str(date.time())[0:5].replace(":", "h") + "m\n")
f = open("PY_report_" + str(datetime.datetime.now().strftime("%d%H%M")) + ".json", "a+") report_file = open("PY_report_" + str(date.strftime("%d%H%M")) + ".json", "a+")
main_loop() main_loop()

27
Ruby/index.rb
View file

@ -25,23 +25,24 @@ def url_generator()
result = PROTOCOLS[rand(0...PROTOCOLS.length)] + '://' result = PROTOCOLS[rand(0...PROTOCOLS.length)] + '://'
url_length = rand(MIN..MAX) url_length = rand(MIN..MAX)
result += rand(36 ** url_length).to_s(36) result += rand(36 ** url_length).to_s(36)
result += DOMAINS[rand(0...DOMAINS.length)] if rand(1...100) <= SECOND result += "." + DOMAINS[rand(0...DOMAINS.length)] if rand(1...100) <= SECOND
result += DOMAINS[rand(0...DOMAINS.length)] result += "." + DOMAINS[rand(0...DOMAINS.length)]
end end
TIMES = ARGV.include?('-t') ? ARGV[ARGV.index('-t') + 1].to_i : 3000 DEFAULTS = JSON.parse(File.read("../defaults.json"))
PROTOCOLS = ARGV.include?('-p') ? ARGV[ARGV.index('-p') + 1].split(",") : ['http'] TIMES = ARGV.include?('-t') ? ARGV[ARGV.index('-t') + 1].to_i : DEFAULTS["times"]
DOMAINS = ARGV.include?('-d') ? ARGV[ARGV.index('-d') + 1].split(",") : ['.co', '.com', '.net', '.edu', '.gov', '.cn', '.org', '.cc', '.us', '.mil', '.ac', '.it', '.de'] PROTOCOLS = ARGV.include?('-p') ? ARGV[ARGV.index('-p') + 1].split(",") : DEFAULTS["protocols"]
SECOND = ARGV.include?('-s') ? ARGV[ARGV.index('-s') + 1].to_i : 1 DOMAINS = ARGV.include?('-d') ? ARGV[ARGV.index('-d') + 1].split(",") : DEFAULTS["domains"]
LOG = ARGV.index('-l').class == Integer SECOND = ARGV.include?('-s') ? ARGV[ARGV.index('-s') + 1].to_i : DEFAULTS["second"]
MIN = ARGV.include?('-min') ? ARGV[ARGV.index('-max') + 1].to_i : 2 LOG = ARGV.include?('-l') ? true : DEFAULTS["log"]
MAX = ARGV.include?('-min') ? ARGV[ARGV.index('-max') + 1].to_i : 50 MIN = ARGV.include?('-min') ? ARGV[ARGV.index('-min') + 1].to_i : DEFAULTS["min"]
MAX = ARGV.include?('-max') ? ARGV[ARGV.index('-max') + 1].to_i : DEFAULTS["max"]
REPORT_FILE = "RB_report_#{Time.new.day}#{Time.new.hour}#{Time.new.min}.json"
DATE = Time.new
puts("\nI am going to look for websites through #{TIMES} random URLs (min length #{MIN} and max length #{MAX}) with the following domains: #{DOMAINS}") puts("\nI am going to look for websites through #{TIMES} random URLs (min length #{MIN} and max length #{MAX}) with the following domains: #{DOMAINS}")
puts("These URLs will use the protocols #{PROTOCOLS} and each of those URLs have #{SECOND} in 100 chance to have a second level domain.") puts("These URLs will use the protocols #{PROTOCOLS} and each of those URLs have #{SECOND} in 100 chance to have a second level domain")
puts("Started at #{Time.new.hour}h#{Time.new.min}m\n") puts("Started at #{DATE.hour}h#{DATE.min}m\n")
REPORT_FILE = "RB_report_#{DATE.day}#{DATE.hour}#{DATE.min}.json"
File.open(REPORT_FILE, 'a+') File.open(REPORT_FILE, 'a+')
main_loop main_loop

30
defaults.json Normal file
View file

@ -0,0 +1,30 @@
{
"times": 2000,
"protocols": [
"http"
],
"domains": [
"com",
"org",
"net",
"tk",
"cn",
"de",
"ru",
"uk",
"nl",
"ca",
"au",
"in",
"ir",
"cc",
"tv",
"live",
"gov",
"edu"
],
"second": 0,
"log": false,
"min": 2,
"max": 15
}