using Crawler.
Utilites;
using Crawler.Extensions.SeleniumExtensions;
using OpenQA.Selenium;
using OpenQA.Selenium.Interactions;
namespace JobsTrackerCrawler
{
class Program
{
static async Task Main(string[] args)
{
bool IsProgramCrash = false;
int retryCount = 0;
int maxRetries = 10;
int maxAttempts = 10;
int attempts = 0;
while (retryCount < maxRetries)
{
try
{
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string folderPath = Path.Combine(documentsPath, "Frase.io");
string downloadsFolder =
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile),
"Downloads");
string successLoggerPath = Path.Combine(folderPath,
"SuccessLog.txt");
string countryAndLanuagePath = Path.Combine(folderPath,
"CountryAndLanguage.txt");
var (country, language) =
GetCountryAndLanguage(countryAndLanuagePath);
var initialFiles =
Directory.GetFiles(downloadsFolder).ToHashSet();
string documentsFolder =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string baseFolder = Path.Combine(documentsFolder, "Frase.io",
"GeneratedContent");
UIConstants.SiteUrl = "https://app.frase.io/app/documents";
SeleniumBrowserMethods.InitializeAndSetupBrowser("ProfileOne");
SeleniumObjectsMethods.PutWaitInThread(2);
if (SeleniumBrowserMethods.GetCurrentUrl() != null &&
SeleniumBrowserMethods.GetCurrentUrl().Contains("login"))
{
SeleniumObjectsMethods.PutWaitInThread(5);
Login();
SeleniumObjectsMethods.PutWaitInThread(5);
}
string[] topics = GetTopicList();
foreach (var topic in topics)
{
if (!CheckInternetConnection())
{
Console.WriteLine("Internet connection lost.
Retrying...");
Thread.Sleep(5000);
continue;
}
else
{
SeleniumBrowserMethods.Navigate(UIConstants.SiteUrl,
true);
}
CreateNewDocumentPopUp();
ChangeCountryAndLanguage(country, language);
SeleniumObjectsMethods.PutWaitInThread(5);
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[placeholder=\"Enter search query\"]", topic, 250);
SeleniumSetMethods.Click(PropertyType.XPath,
"//button[span[text()='Start']]", 250);
while (attempts < maxAttempts && !
SeleniumObjectsMethods.IsElementVisible(PropertyType.XPath, "//button[@role='tab'
and contains(@aria-controls, 'radix') and contains(text(), 'Outline')]"))
{
SeleniumObjectsMethods.PutWaitInThread(3); // Wait for
3 seconds
attempts++;
if (attempts >= maxAttempts)
{
Console.WriteLine("Max attempts reached: Element
not found.");
break; // Stop the loop after 10 tries
}
}
SeleniumObjectsMethods.PutWaitInThread(5);
SeleniumBrowserMethods.RefreshPage();
SeleniumObjectsMethods.PutWaitInThread(2);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[aria-controls*=\"-content-outline\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[class*=\"backdrop-blur-lg bg-emerald-200/30 dark:bg-emerald-800/30\"]",
250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[id=\"optimize-tab\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[data-orientation=\"horizontal\"][data-state=\"active\"] button[aria-
haspopup=\"menu\"][aria-expanded=\"false\"][data-state=\"closed\"]", 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"a[download=\"topics.csv\"]", 250);
WaitForNewFiles(downloadsFolder, initialFiles, 600, topic);
SeleniumObjectsMethods.PutWaitInThread(5);
string topicFolder = Path.Combine(baseFolder, topic);
string latestCsvFile = GetLatestFile(downloadsFolder,
"*.csv");
if (!Directory.Exists(topicFolder))
{
Directory.CreateDirectory(topicFolder);
}
if (!string.IsNullOrEmpty(latestCsvFile))
{
string newCsvFilePath = Path.Combine(topicFolder, topic
+ ".csv");
if (File.Exists(newCsvFilePath))
{
File.Delete(newCsvFilePath);
Console.WriteLine($"Existing CSV file deleted:
{newCsvFilePath}");
}
SeleniumObjectsMethods.PutWaitInThread(4);
File.Move(latestCsvFile, newCsvFilePath);
}
else
{
Console.WriteLine("No CSV file found in Downloads.");
}
File.AppendAllText(successLoggerPath, $"'{topic}' has been
successfully generated by AI.{Environment.NewLine}");
Console.WriteLine($"The topic '{topic}' has been
successfully generated by AI.");
RemoveTopicFromFile(topic);
}
// If execution reaches here, reset retryCount
retryCount = 0;
break;
}
catch (Exception ex)
{
retryCount++;
Console.WriteLine($"Attempt {retryCount} failed:
{ex.Message}");
LogError("An unknown error occurred in the Scraper: " +
ex.Message + "\nStackTrace: " + ex.StackTrace);
if (retryCount >= maxRetries)
{
Console.WriteLine("Maximum retry attempts reached. Exiting
program.");
break;
}
Console.WriteLine("Retrying after 10 seconds...");
Thread.Sleep(10000); // Wait before retrying
}
finally
{
SeleniumBrowserMethods.DisposeBrowser();
}
}
}
public static void RemoveTopicFromFile(string completedTopic)
{
try
{
string userDocumentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string topicListPath = Path.Combine(userDocumentsPath, "Frase.io",
"TopicList.txt");
if (File.Exists(topicListPath))
{
var topics = File.ReadAllLines(topicListPath)
.Where(line => !
string.IsNullOrWhiteSpace(line) && line.Trim() != completedTopic)
.ToList();
File.WriteAllLines(topicListPath, topics);
Console.WriteLine($"Removed completed topic:
{completedTopic}");
}
}
catch (Exception ex)
{
Console.WriteLine($"Error removing topic: {ex.Message}");
}
}
public static bool CheckInternetConnection()
{
try
{
using (var client = new System.Net.WebClient())
using (client.OpenRead("http://www.google.com"))
{
return true;
}
}
catch
{
return false;
}
}
static (string Email, string Password) Credentials()
{
// Get the Documents folder path dynamically
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
string folderPath = Path.Combine(documentsPath, "Frase.io");
string filePath = Path.Combine(folderPath, "Credentials.txt");
try
{
// Check if the file exists
if (File.Exists(filePath))
{
// Read all lines from the file
string[] lines = File.ReadAllLines(filePath);
// Ensure the file has at least two lines
if (lines.Length >= 2 && !string.IsNullOrWhiteSpace(lines[0])
&& !string.IsNullOrWhiteSpace(lines[1]))
{
return (lines[0].Trim(), lines[1].Trim()); // Return email
and password
}
else
{
Console.WriteLine("Error: The file does not contain a valid
email and password.");
}
}
else
{
Console.WriteLine("Error: Credentials file not found.");
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
}
return (string.Empty, string.Empty); // Return empty values if an error
occurs
}
static void Login()
{
SeleniumObjectsMethods.PutWaitInThread(3);
var (email, password) = Credentials();
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[type='email']", email, 250);
SeleniumSetMethods.EnterText(PropertyType.CssSelector,
"input[type='password'", password, 250);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"button[type=\"submit\"]", 250);
SeleniumObjectsMethods.PutWaitInThread(1);
}
static void CreateNewDocumentPopUp()
{
SeleniumObjectsMethods.PutWaitInThread(5);
//click on new document button
SeleniumSetMethods.Click(PropertyType.XPath,
"(//button[.//span[text()='New document']])[1]", 250);
SeleniumSetMethods.Click(PropertyType.XPath, "//span[text()='Blank
Document']/ancestor::div[contains(@class, 'transition-all')]", 250);
SeleniumSetMethods.Click(PropertyType.XPath,
"//button[span[text()='Continue']]", 250);
}
static void ChangeCountryAndLanguage(string country, string language)
{
SeleniumObjectsMethods.PutWaitInThread(3);
if (!string.IsNullOrEmpty(country) && !string.IsNullOrEmpty(language))
{
SeleniumSetMethods.Click(PropertyType.XPath,
"//input[@placeholder='Enter search query']/following-sibling::div/button[2]", 60);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div[class^=\"space-y-2\"]:nth-child(2) button[aria-
haspopup=\"dialog\"]:nth-child(2)");
var countriesNames =
SeleniumGetMethods.GetWebElements(PropertyType.CssSelector, "div[role='group'] >
div");
foreach (var countriesName in countriesNames)
{
if
(SeleniumGetMethods.GetInnerTextFromHTML(countriesName).ToLower().Contains(country.
ToLower()))
{
SeleniumSetMethods.Click(countriesName);
Actions actions = new Actions(PropertiesCollection.Driver);
actions.SendKeys(Keys.Escape).Perform();
break;
}
}
SeleniumObjectsMethods.PutWaitInThread(2);
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div[class^=\"space-y-2\"]:nth-child(3) button[aria-
haspopup=\"dialog\"]:nth-child(2)", 60);
var languageNames =
SeleniumGetMethods.GetWebElements(PropertyType.CssSelector, "div[role='group'] >
div");
foreach (var name in languageNames)
{
if
(SeleniumGetMethods.GetInnerTextFromHTML(name).ToLower().Contains(language.ToLower(
)))
{
SeleniumSetMethods.Click(name);
Actions actions = new Actions(PropertiesCollection.Driver);
actions.SendKeys(Keys.Escape).Perform();
break;
}
}
SeleniumSetMethods.Click(PropertyType.CssSelector,
"div[role=\"dialog\"] form > div:nth-child(4) > button:nth-child(2)");
}
static void LogError(string message)
{
// Get the Documents folder path dynamically
string documentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
// Define the log file path dynamically
string logFilePath = Path.Combine(documentsPath, "Frase.io",
"ErrorLog.txt");
// Ensure the directory exists
Directory.CreateDirectory(Path.GetDirectoryName(logFilePath));
// Write the error message to the log file
File.AppendAllText(logFilePath, $"{DateTime.Now}: {message}
{Environment.NewLine}");
Console.WriteLine("Error logged successfully.");
}
public static string[] GetTopicList()
{
try
{
// Get the user's documents folder
string userDocumentsPath =
Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
// Construct the path to the TopicList.txt file
string topicListPath = Path.Combine(userDocumentsPath, "Frase.io",
"TopicList.txt");
// Check if the file exists
if (File.Exists(topicListPath))
{
// Read the topics from the file, remove empty and whitespace-
only lines
string[] topics = File.ReadAllLines(topicListPath)
.Where(line => !
string.IsNullOrWhiteSpace(line)) // Filters out empty lines
.ToArray();
return topics;
}
else
{
Console.WriteLine("TopicList.txt file does not exist.");
return Array.Empty<string>(); // Return an empty array if file
does not exist
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
return Array.Empty<string>(); // Return an empty array in case of
an error
}
}
static string GetLatestFile(string folderPath, string searchPattern)
{
try
{
var directory = new DirectoryInfo(folderPath);
var file = directory.GetFiles(searchPattern)
.OrderByDescending(f => f.LastWriteTime)
.FirstOrDefault();
return file?.FullName;
}
catch (Exception ex)
{
Console.WriteLine($"Error getting latest file: {ex.Message}");
return null;
}
}
static void WaitForNewFiles(string folderPath, HashSet<string>
initialFiles, int timeoutInSeconds, string topicName)
{
DateTime endTime = DateTime.Now.AddSeconds(timeoutInSeconds);
while (DateTime.Now < endTime)
{
var allFiles = Directory.GetFiles(folderPath);
var newFiles = allFiles.Where(f => !
initialFiles.Contains(f)).ToList();
var downloadingFiles = newFiles.Where(f =>
f.EndsWith(".crdownload") || f.EndsWith(".part")).ToList();
var csvFiles = newFiles.Where(f => f.EndsWith(".csv")).ToList();
if (csvFiles.Any() && !downloadingFiles.Any())
{
Console.WriteLine("File downloaded successfully! Waiting for
file to be fully written...");
string downloadedFile = csvFiles.First();
// Wait until the file is no longer locked
int retries = 5;
while (retries > 0)
{
try
{
using (FileStream fs = new FileStream(downloadedFile,
FileMode.Open, FileAccess.Read, FileShare.None))
{
Console.WriteLine("File is fully available.");
return;
}
}
catch (IOException)
{
Console.WriteLine("File is still being written,
retrying...");
Thread.Sleep(2000);
retries--;
}
}
Console.WriteLine("Warning: File may still be incomplete.");
return;
}
Thread.Sleep(1000);
}
LogError($"Files did not download for topic = '{topicName}'");
}
public static (string Country, string Language)
GetCountryAndLanguage(string filePath)
{
if (File.Exists(filePath))
{
string[] lines = File.ReadAllLines(filePath);
if (lines.Length >= 2)
return (lines[0].Trim(), lines[1].Trim());
}
return (string.Empty, string.Empty); // Return empty values if file is
missing or invalid
}
}
}
