10000 General code refactoring and adding api http urls counting functionality · Dyzio18/java-web-bot-library@e68c5ee · GitHub
[go: up one dir, main page]
Skip to content

Commit e68c5ee

Browse files
xBlackyPLxBlackyPL
committed
General code refactoring and adding api http urls counting functionality
1 parent f015134 commit e68c5ee

29 files changed

+56
-15
lines changed

‎src/main/java/stdBot/App.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ public static void main( String[] args )
77
{
88
RemoteMapper remoteMapper = new RemoteMapper();
99

10-
remoteMapper.start("http://www.wggios.agh.edu.pl/", CollectorTypeEnum.Map, 1);
10+
remoteMapper.start("http://www.wggios.agh.edu.pl/", CollectorTypeEnum.Map, 2);
1111

1212
SiteMap siteMap = remoteMapper.getResult();
1313

1414
siteMap.print();
15+
siteMap.printInternalLinksNumber();
16+
siteMap.printRelationGraph();
1517
}
1618
}

‎src/main/java/stdBot/HyperLinksCollector.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22

33
public class HyperLinksCollector extends RemoteCollector {
44

5-
public HyperLinksCollector(RemoteHandler handler, String address, int maxRecurtionLevel) {
6-
super(handler, address, maxRecurtionLevel);
5+
public HyperLinksCollector(RemoteHandler handler, String address, int maxRecursionLevel) {
6+
super(handler, address, maxRecursionLevel);
77
collectableRemotely = factory.create(MapTypeEnum.HyperLinksMap);
88
}
99
}

‎src/main/java/stdBot/RemoteCollector.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,25 +6,24 @@ public abstract class RemoteCollector extends Thread {
66
protected CollectableRemotely collectableRemotely = null;
77
private String address;
88
private RemoteHandler handler;
9-
protected int maxRecurtionLevel;
9+
protected int maxRecursionLevel;
1010

11-
public RemoteCollector(RemoteHandler handler, String address, int maxRecurtionLevel){
11+
public RemoteCollector(RemoteHandler handler, String address, int maxRecursionLevel){
1212
this.handler = handler;
1313
this.address = address;
14-
this.maxRecurtionLevel = maxRecurtionLevel;
14+
this.maxRecursionLevel = maxRecursionLevel;
1515
}
1616

1717
@Override
1818
public void run(){
1919
collectableRemotely.setEntryPoint(address);
20-
collectableRemotely.setMaxRecursionLevel(maxRecurtionLevel);
20+
collectableRemotely.setMaxRecursionLevel(maxRecursionLevel);
2121

2222
while (!collectableRemotely.hasFinished())
2323
collectableRemotely.collectOne(handler);
2424
}
2525

2626
public SiteMap collectAll(){
27-
2827
try {
2928
join();
3029
} catch (InterruptedException e) {

‎src/main/java/stdBot/RemoteMapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,9 @@ public class RemoteMapper {
66
private RemoteCollector collector;
77
private RemoteHandler handler;
88

9-
public void start(String address, CollectorTypeEnum collectorTypeEnum, int maxRecurtionLevel){
9+
public void start(String address, CollectorTypeEnum collectorTypeEnum, int maxRecursionLevel){
1010
handler = remoteHandlersFactory.create(address);
11-
collector = collectorsFactory.create(collectorTypeEnum, handler, address, maxRecurtionLevel);
11+
collector = collectorsFactory.create(collectorTypeEnum, handler, address, maxRecursionLevel);
1212
collector.start();
1313
}
1414

‎src/main/java/stdBot/RemoteSiteMap.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ public class RemoteSiteMap implements CollectableRemotely {
1010
private Queue<String> currentLinks = new ArrayDeque<>();
1111
private Queue<String> nextLinks = new ArrayDeque<>();
1212
private SiteMap siteMap = new SiteMap();
13-
private int currentRecurtionLevel = 0;
14-
private int maxRecurtionLevel = 0;
13+
private int currentRecursionLevel = 0;
14+
private int maxRecursionLevel = 0;
1515
private Pattern pattern = Pattern.compile("href=\".*?\"");
1616
private String domain;
1717

@@ -64,12 +64,12 @@ public SiteMap getMap() {
6464

6565
@Override
6666
public boolean hasFinished() {
67-
return currentRecurtionLevel > maxRecurtionLevel || (currentLinks.isEmpty() && nextLinks.isEmpty());
67+
return currentRecursionLevel > maxRecursionLevel || (currentLinks.isEmpty() && nextLinks.isEmpty());
6868
}
6969

7070
@Override
7171
public void setMaxRecursionLevel(int level) {
72-
maxRecurtionLevel = level;
72+
maxRecursionLevel = level;
7373
}
7474

7575
private void enterSite(RemoteHandler handler, String address){
@@ -80,7 +80,7 @@ private String getNextAddress(){
8080
if (currentLinks.isEmpty()){
8181
currentLinks = nextLinks;
8282
nextLinks = new ArrayDeque<>();
83-
currentRecurtionLevel++;
83+
currentRecursionLevel++;
8484
}
8585
return currentLinks.poll();
8686
}

‎src/main/java/stdBot/SiteMap.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
import javafx.util.Pair;
44

5+
import java.util.HashMap;
56
import java.util.HashSet;
7+
import java.util.Map;
68
import java.util.Set;
79

810
public class SiteMap {
@@ -33,4 +35,26 @@ public String toHTML() {
3335
return stringBuilder.toString();
3436

3537
}
38+
39+
public void printInternalLinksNumber() {
40+
System.out.println("Total number of internal links is: " + knownInternalLinks.size());
41+
}
42+
43+
public void printRelationGraph() {
44+
Set<String> keys = new HashSet<>();
45+
46+
for (Pair<String, String> relation: knownRelations) {
47+
keys.add(relation.getKey());
48+
}
49+
50+
for (String key:
51+
keys) {
52+
System.out.println("\t" + key);
53+
printRelationForEveryKey(key);
54+
}
55+
}
56+
57+
public void printRelationForEveryKey(String key) {
58+
59+
}
3660
}

‎stdBot.iml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
3+
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
4+
<output url="file://$MODULE_DIR$/target/classes" />
5+
<output-test url="file://$MODULE_DIR$/target/test-classes" />
6+
<content url="file://$MODULE_DIR$">
7+
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
8+
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
9+
<excludeFolder url="file://$MODULE_DIR$/target" />
10+
</content>
11+
<orderEntry type="inheritedJdk" />
12+
<orderEntry type="sourceFolder" forTests="false" />
13+
<orderEntry type="library" scope="TEST" name="Maven: junit:junit:4.11" level="project" />
14+
<orderEntry type="library" scope="TEST" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
15+
</component>
16+
</module>

‎target/classes/Demo.class

3.89 KB
Binary file not shown.

‎target/classes/demo/Common.class

1.64 KB
Binary file not shown.

‎target/classes/stdBot/AddressUtility.class

1.29 KB
Binary file not shown.

0 commit comments

Comments
 (0)
0