8000 trying new fs keyviz stuff · coderatgoogle/java-docs-samples@96a17ca · GitHub
[go: up one dir, main page]
Skip to content

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 96a17ca

Browse files
billyjacobsonbillyjacobson
committed
trying new fs keyviz stuff
1 parent 8fb36e7 commit 96a17ca

File tree

1 file changed

+67
-56
lines changed

1 file changed

+67
-56
lines changed

‎bigtable/beam/keyviz-art/src/main/java/keyviz/FSReadData2.java

Lines changed: 67 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ public static void main(String[] args) {
4343

4444
Pipeline p = Pipeline.create(options);
4545
// Initiates a new pipeline every second
46-
p.apply(GenerateSequence.from(0).withRate(1, new Duration(1)))
46+
p.apply(GenerateSequence.from(0).withRate(1, new Duration(1000)))
4747
.apply(ParDo.of(new ReadFromTableFn()));
4848

4949
p.run();
@@ -58,7 +58,8 @@ public static class ReadFromTableFn extends DoFn<Long, Void> {
5858
protected Connection connection;
5959
protected Datastore datastore;
6060
private KeyFactory keyFactory;
61-
private final String kind = "BillyDF500reversed1mswithP";
61+
// private final String kind = "BillyDF500reversed1mswithP";
62+
private final String kind = "BillyDF500reversed1s";
6263

6364
private static final int RANDOM_ID_BOUND = 500;
6465
Random rand = new Random();
@@ -93,70 +94,80 @@ public void processElement(@Element Long count, OutputReceiver<Void> out) {
9394
String numberFormat = "%0" + maxLength + "d";
9495
Datastore ds = getDatastore();
9596

96-
CompositeFilter evenRangeFilter = CompositeFilter
97-
.and(
98-
PropertyFilter.lt("__key__", keyFactory.newKey("250"))
99-
// PropertyFilter.le("__key__", "1"),
100-
// PropertyFilter.ge("__key__", "2"), PropertyFilter.le("__key__", "3"),
101-
// PropertyFilter.ge("__key__", "4"), PropertyFilter.le("__key__", "5"),
102-
// PropertyFilter.ge("__key__", "6"), PropertyFilter.le("__key__", "7"),
103-
// PropertyFilter.ge("__key__", "8"), PropertyFilter.le("__key__", "9")
104-
);
105-
106-
CompositeFilter oddRangeFilter = CompositeFilter
107-
.and(
108-
PropertyFilter.ge("__key__", keyFactory.newKey("250"))
109-
// , PropertyFilter.le("__key__", ""),
110-
// PropertyFilter.ge("__key__", "3"), PropertyFilter.le("__key__", "4"),
111-
// PropertyFilter.ge("__key__", "5"), PropertyFilter.le("__key__", "6"),
112-
// PropertyFilter.ge("__key__", "7"), PropertyFilter.le("__key__", "8"),
113-
// PropertyFilter.ge("__key__", "9")
114-
);
97+
// CompositeFilter evenRangeFilter = CompositeFilter
98+
// .and(
99+
// PropertyFilter.lt("__key__", keyFactory.newKey("250"))
100+
// // PropertyFilter.le("__key__", "1"),
101+
// // PropertyFilter.ge("__key__", "2"), PropertyFilter.le("__key__", "3"),
102+
// // PropertyFilter.ge("__key__", "4"), PropertyFilter.le("__key__", "5"),
103+
// // PropertyFilter.ge("__key__", "6"), PropertyFilter.le("__key__", "7"),
104+
// // PropertyFilter.ge("__key__", "8"), PropertyFilter.le("__key__", "9")
105+
// );
106+
//
107+
// CompositeFilter oddRangeFilter = CompositeFilter
108+
// .and(
109+
// PropertyFilter.ge("__key__", keyFactory.newKey("250"))
110+
// // , PropertyFilter.le("__key__", ""),
111+
// // PropertyFilter.ge("__key__", "3"), PropertyFilter.le("__key__", "4"),
112+
// // PropertyFilter.ge("__key__", "5"), PropertyFilter.le("__key__", "6"),
113+
// // PropertyFilter.ge("__key__", "7"), PropertyFilter.le("__key__", "8"),
114+
// // PropertyFilter.ge("__key__", "9")
115+
// );
115116

116117
EntityQuery.Builder query = Query.newEntityQueryBuilder()
10000
117118
.setKind(kind);
118-
if ((count / (2 * 60 * 1000)) % 2 == 0) {
119-
query.setFilter(oddRangeFilter);
120-
} else {
121-
query.setFilter(evenRangeFilter);
122-
}
119+
120+
// if ((count / (2 * 60 * 1000)) % 2 == 0) {
121+
// query.setFilter(oddRangeFilter);
122+
// } else {
123+
// query.setFilter(evenRangeFilter);
124+
// }
123125
QueryResults<Entity> entityQueryResults = ds.run(query.build());
124126
int c = 0;
125-
while (entityQueryResults.hasNext()) {
126-
Entity entity = entityQueryResults.next();
127-
System.out.println(entity);
128-
c++;
129-
}
130-
System.out.println(c + " rows queried");
131-
127+
// while (entityQueryResults.hasNext()) {
128+
// Entity entity = entityQueryResults.next();
129+
// System.out.println(entity);
130+
// c++;
131+
// }
132132

133133
// for (int i = 0; i < maxInput; i++) {
134-
//
135134
// String paddedRowkey = String.format(numberFormat, i);
136135
// String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
137-
// int rowNumber = i / rowHeight;
138-
//
139-
// float p = (float) rowNumber / numRows;
140-
// System.out.printf("index: %d rowNumber %d prob %f count %d", i, rowNumber, p, count);
141-
//
142-
// if (Math.random() <= p || count < 3000) {
143-
// FullEntity<com.google.cloud.datastore.Key> fullEntity =
144-
// com.google.cloud.datastore.Entity.newBuilder(keyFactory.newKey(reversedRowkey))
145-
// .set("description", getRandomStringValue())
146-
// .build();
147-
// entities.add(fullEntity);
148-
//
149-
// // logger.info("writing entity" + fullEntity.getKey());
150-
//
151-
// }
152-
// // 500 per write limit
153-
// if (i % 500 == 0) {
154-
// System.out.printf("500 mod 0");
155-
// ds.put(entities.toArray(new FullEntity<?>[0]));
156-
// entities = new ArrayList<>();
157-
// }
136+
// datastore.get(keyFactory.newKey(reversedRowkey));
137+
// c++;
158138
// }
159-
// ds.put(entities.toArray(new FullEntity<?>[0]));
139+
// System.out.println(c + " entities fetched");
140+
141+
142+
for (int i = 0; i < maxInput; i++) {
143+
144+
String paddedRowkey = String.format(numberFormat, i);
145+
String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
146+
int rowNumber = i / rowHeight;
147+
148+
float p = (float) rowNumber / numRows;
149+
System.out.printf("index: %d rowNumber %d prob %f count %d", i, rowNumber, p, count);
150+
151+
if (Math.random() <= p || count < 120) {
152+
FullEntity<com.google.cloud.datastore.Key> fullEntity =
153+
com.google.cloud.datastore.Entity.newBuilder(keyFactory.newKey(reversedRowkey))
154+
.set("description", getRandomStringValue())
155+
.build();
156+
entities.add(fullEntity);
157+
// logger.info("writing entity" + fullEntity.getKey());
158+
c++;
159+
}
160+
// 500 per write limit
161+
if (i % 500 == 0) {
162+
System.out.printf("500 mod 0");
163+
ds.put(entities.toArray(new FullEntity<?>[0]));
164+
entities = new ArrayList<>();
165+
}
166+
}
167+
168+
System.out.println(c + " entities updated");
169+
170+
ds.put(entities.toArray(new FullEntity<?>[0]));
160171

161172
// String kind = "Billy10ms1krs";
162173
// int maxInput = 1000;

0 commit comments

Comments
 (0)
0