knowledgepixels/nanopub-query | Build 17945178198 | com/knowledgepixels/query/MainVerticle.java | Coveralls

1

package com.knowledgepixels.query;

2

3

import java.io.InputStream;

4

import java.net.URLEncoder;

5

import java.util.ArrayList;

6

import java.util.Collections;

7

import java.util.List;

8

import java.util.Scanner;

9

import java.util.concurrent.Executors;

10

import java.util.concurrent.TimeUnit;

11

12

import org.eclipse.rdf4j.model.Value;

13

import org.slf4j.Logger;

14

import org.slf4j.LoggerFactory;

15

16

import com.github.jsonldjava.shaded.com.google.common.base.Charsets;

17

import com.knowledgepixels.query.GrlcSpec.InvalidGrlcSpecException;

18

19

import io.micrometer.prometheus.PrometheusMeterRegistry;

20

import io.vertx.core.AbstractVerticle;

21

import io.vertx.core.Future;

22

import io.vertx.core.Promise;

23

import io.vertx.core.buffer.Buffer;

24

import io.vertx.core.http.HttpClient;

25

import io.vertx.core.http.HttpClientOptions;

26

import io.vertx.core.http.HttpMethod;

27

import io.vertx.core.http.HttpServer;

28

import io.vertx.core.http.HttpServerResponse;

29

import io.vertx.core.http.PoolOptions;

30

import io.vertx.ext.web.Router;

31

import io.vertx.ext.web.RoutingContext;

32

import io.vertx.ext.web.handler.CorsHandler;

33

import io.vertx.ext.web.handler.StaticHandler;

34

import io.vertx.ext.web.proxy.handler.ProxyHandler;

35

import io.vertx.httpproxy.Body;

36

import io.vertx.httpproxy.HttpProxy;

37

import io.vertx.httpproxy.ProxyContext;

38

import io.vertx.httpproxy.ProxyInterceptor;

39

import io.vertx.httpproxy.ProxyRequest;

40

import io.vertx.httpproxy.ProxyResponse;

41

import io.vertx.micrometer.PrometheusScrapingHandler;

42

import io.vertx.micrometer.backends.BackendRegistries;

43

44

/**

45

 * Main verticle that coordinates the incoming HTTP requests.

46

*/

47

@GeneratedFlagForDependentElements

48

public class MainVerticle extends AbstractVerticle {

49

50

    private static String css = null;

51

52

    private static final Logger log = LoggerFactory.getLogger(MainVerticle.class);

53

54

/**

55

     * Start the main verticle.

56

57

     * @param startPromise the promise to complete when the verticle is started

58

     * @throws Exception if an error occurs during startup

59

*/

60

    @Override

61

    public void start(Promise<Void> startPromise) throws Exception {

62

        HttpClient httpClient = vertx.createHttpClient(

63

                new HttpClientOptions()

64

                        .setConnectTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_CONNECT_TIMEOUT", 1000))

65

                        .setIdleTimeoutUnit(TimeUnit.SECONDS)

66

                        .setIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))

67

                        .setReadIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))

68

                        .setWriteIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60)),

69

                new PoolOptions().setHttp1MaxSize(200).setHttp2MaxSize(200)

70

);

71

72

        HttpServer proxyServer = vertx.createHttpServer();

73

        Router proxyRouter = Router.router(vertx);

74

        proxyRouter.route().handler(CorsHandler.create().addRelativeOrigin(".*"));

75

76

        // Metrics

77

        final var metricsHttpServer = vertx.createHttpServer();

78

        final var metricsRouter = Router.router(vertx);

79

        metricsHttpServer.requestHandler(metricsRouter).listen(9394);

80

81

        final var metricsRegistry = (PrometheusMeterRegistry) BackendRegistries.getDefaultNow();

82

        final var collector = new MetricsCollector(metricsRegistry);

83

        metricsRouter.route("/metrics").handler(PrometheusScrapingHandler.create(metricsRegistry));

84

        // ----------

85

        // This part is only used if the redirection is not done through Nginx.

86

        // See nginx.conf and this bug report: https://github.com/eclipse-rdf4j/rdf4j/discussions/5120

87

        HttpProxy rdf4jProxy = HttpProxy.reverseProxy(httpClient);

88

        String proxy = Utils.getEnvString("RDF4J_PROXY_HOST", "rdf4j");

89

        int proxyPort = Utils.getEnvInt("RDF4J_PROXY_PORT", 8080);

90

        rdf4jProxy.origin(proxyPort, proxy);

91

93

94

            @Override

95

            @GeneratedFlagForDependentElements

96

            public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {

97

                ProxyRequest request = context.request();

98

                request.setURI(request.getURI().replaceAll("/", "_").replaceFirst("^_repo_", "/rdf4j-server/repositories/"));

99

                // For later to try to get HTML tables out:

100

//                                if (request.headers().get("Accept") == null) {

101

//                                        request.putHeader("Accept", "text/html");

102

//                                }

103

//                                request.putHeader("Accept", "application/json");

104

                return ProxyInterceptor.super.handleProxyRequest(context);

105

106

107

            @Override

108

            @GeneratedFlagForDependentElements

109

            public Future<Void> handleProxyResponse(ProxyContext context) {

110

                ProxyResponse resp = context.response();

111

                resp.putHeader("Access-Control-Allow-Origin", "*");

112

                resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");

113

                // For later to try to get HTML tables out:

114

//                                String acceptHeader = context.request().headers().get("Accept");

115

//                                if (acceptHeader != null && acceptHeader.contains("text/html")) {

116

//                                        resp.putHeader("Content-Type", "text/html");

117

//                                        resp.setBody(Body.body(Buffer.buffer("<html><body><strong>test</strong></body></html>")));

118

//                                }

119

                return ProxyInterceptor.super.handleProxyResponse(context);

120

121

122

});

123

        // ----------

124

125

        proxyRouter.route(HttpMethod.GET, "/repo").handler(req -> handleRedirect(req, "/repo"));

126

        proxyRouter.route(HttpMethod.GET, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

127

        proxyRouter.route(HttpMethod.POST, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

128

        proxyRouter.route(HttpMethod.HEAD, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

129

        proxyRouter.route(HttpMethod.OPTIONS, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

130

        proxyRouter.route(HttpMethod.GET, "/tools/*").handler(req -> {

131

            final String yasguiPattern = "^/tools/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?/yasgui\\.html$";

132

            if (req.normalizedPath().matches(yasguiPattern)) {

133

                String repo = req.normalizedPath().replaceFirst(yasguiPattern, "$1$2");

134

                req.response()

135

                        .putHeader("content-type", "text/html")

136

                        .end("<!DOCTYPE html>\n"

137

                                + "<html lang=\"en\">\n"

138

                                + "<head>\n"

139

                                + "<meta charset=\"utf-8\">\n"

140

                                + "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"

141

                                + "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"

142

                                + "<title>Nanopub Query SPARQL Editor for repository: " + repo + "</title>\n"

143

                                + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

144

                                + "<link href='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.css' rel='stylesheet' type='text/css'/>\n"

145

                                + "<style>.yasgui .endpointText {display:none !important;}</style>\n"

146

                                + "<script type=\"text/javascript\">localStorage.clear();</script>\n"

147

                                + "</head>\n"

148

                                + "<body>\n"

149

                                + "<h3>Nanopub Query SPARQL Editor for repository: " + repo + "</h3>\n"

150

                                + "<div id='yasgui'></div>\n"

151

                                + "<script src='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.js'></script>\n"

152

                                + "<script type=\"text/javascript\">\n"

153

                                + "var yasgui = YASGUI(document.getElementById(\"yasgui\"), {\n"

154

                                + "  yasqe:{sparql:{endpoint:'/repo/" + repo + "'},value:'" + Utils.defaultQuery.replaceAll("\n", "\\\\n") + "'}\n"

155

                                + "});\n"

156

                                + "</script>\n"

157

                                + "</body>\n"

158

                                + "</html>");

159

            } else {

160

                req.response()

161

                        .putHeader("content-type", "text/plain")

162

                        .setStatusCode(404)

163

                        .end("not found");

164

165

});

166

        proxyRouter.route(HttpMethod.GET, "/page").handler(req -> handleRedirect(req, "/page"));

167

        proxyRouter.route(HttpMethod.GET, "/page/*").handler(req -> {

168

            final String pagePattern = "^/page/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?$";

169

            if (req.normalizedPath().matches(pagePattern)) {

170

                String repo = req.normalizedPath().replaceFirst(pagePattern, "$1$2");

171

                req.response()

172

                        .putHeader("content-type", "text/html")

173

                        .end("<!DOCTYPE html>\n"

174

                                + "<html lang=\"en\">\n"

175

                                + "<head>\n"

176

                                + "<meta charset=\"utf-8\">\n"

177

                                + "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"

178

                                + "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"

179

                                + "<title>Nanopub Query repo: " + repo + "</title>\n"

180

                                + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

181

                                + "</head>\n"

182

                                + "<body>\n"

183

                                + "<h3>Nanopub Query repo: " + repo + "</h3>\n"

184

                                + "<p>Endpoint: <a href=\"/repo/" + repo + "\">/repo/" + repo + "</a></p>"

185

                                + "<p>YASGUI: <a href=\"/tools/" + repo + "/yasgui.html\">/tools/" + repo + "/yasgui.hml</a></p>"

186

                                + "</body>\n"

187

                                + "</html>");

188

            } else {

189

                req.response()

190

                        .putHeader("content-type", "text/plain")

191

                        .setStatusCode(404)

192

                        .end("not found");

193

194

});

195

        proxyRouter.route(HttpMethod.GET, "/").handler(req -> {

196

            String repos = "";

197

            List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

198

            Collections.sort(repoList);

199

            for (String s : repoList) {

200

                if (s.startsWith("pubkey_") || s.startsWith("type_")) continue;

201

                repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a></code></li>";

202

203

            String pinnedApisValue = Utils.getEnvString("NANOPUB_QUERY_PINNED_APIS", "");

204

            String[] pinnedApis = pinnedApisValue.split(" ");

205

            String pinnedApiLinks = "";

206

            if (!pinnedApisValue.isEmpty()) {

207

                for (String s : pinnedApis) {

208

                    pinnedApiLinks = pinnedApiLinks + "<li><a href=\"openapi/?url=spec/" + s + "%3Fapi-version=latest\">" + s.replaceFirst("^.*/", "") + "</a></li>";

209

210

                pinnedApiLinks = "<p>Pinned APIs:</p>\n" +

211

                        "<ul>\n" +

212

                        pinnedApiLinks +

213

                        "</ul>\n";

214

215

            req.response()

216

                    .putHeader("content-type", "text/html")

217

                    .end("<!DOCTYPE html>\n"

218

                            + "<html lang='en'>\n"

219

                            + "<head>\n"

220

                            + "<title>Nanopub Query</title>\n"

221

                            + "<meta charset='utf-8'>\n"

222

                            + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

223

                            + "</head>\n"

224

                            + "<body>\n"

225

                            + "<h1>Nanopub Query</h1>"

226

                            + "<p>General repos:</p>"

227

                            + "<ul>" + repos + "</ul>"

228

                            + "<p>Specific repos:</p>"

229

                            + "<ul>"

230

                            + "<li><a href=\"/pubkeys\">Pubkey Repos</a></li>"

231

                            + "<li><a href=\"/types\">Type Repos</a></li>"

232

                            + "</ul>"

233

                            + pinnedApiLinks

234

                            + "</body>\n"

235

                            + "</html>");

236

});

237

        proxyRouter.route(HttpMethod.GET, "/pubkeys").handler(req -> {

238

            String repos = "";

239

            List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

240

            Collections.sort(repoList);

241

            for (String s : repoList) {

242

                if (!s.startsWith("pubkey_")) continue;

243

                String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");

244

                Value hashObj = Utils.getObjectForHash(hash);

245

                String label;

246

                if (hashObj == null) {

247

                    label = "";

248

                } else {

249

                    label = " (" + Utils.getShortPubkeyName(hashObj.stringValue()) + ")";

250

251

                s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");

252

                repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";

253

254

            req.response()

255

                    .putHeader("content-type", "text/html")

256

                    .end("<!DOCTYPE html>\n"

257

                            + "<html lang='en'>\n"

258

                            + "<head>\n"

259

                            + "<title>Nanopub Query: Pubkey Repos</title>\n"

260

                            + "<meta charset='utf-8'>\n"

261

                            + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

262

                            + "</head>\n"

263

                            + "<body>\n"

264

                            + "<h3>Pubkey Repos</h3>"

265

                            + "<p>Repos:</p>"

266

                            + "<ul>" + repos + "</ul>"

267

                            + "</body>\n"

268

                            + "</html>");

269

});

270

        proxyRouter.route(HttpMethod.GET, "/types").handler(req -> {

271

            String repos = "";

272

            List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

273

            Collections.sort(repoList);

274

            for (String s : repoList) {

275

                if (!s.startsWith("type_")) continue;

276

                String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");

277

                Value hashObj = Utils.getObjectForHash(hash);

278

                String label;

279

                if (hashObj == null) {

280

                    label = "";

281

                } else {

282

                    label = " (" + hashObj.stringValue() + ")";

283

284

                s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");

285

                repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";

286

287

            req.response()

288

                    .putHeader("content-type", "text/html")

289

                    .end("<!DOCTYPE html>\n"

290

                            + "<html lang='en'>\n"

291

                            + "<head>\n"

292

                            + "<title>Nanopub Query: Type Repos</title>\n"

293

                            + "<meta charset='utf-8'>\n"

294

                            + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

295

                            + "</head>\n"

296

                            + "<body>\n"

297

                            + "<h3>Type Repos</h3>"

298

                            + "<p>Repos:</p>"

299

                            + "<ul>" + repos + "</ul>"

300

                            + "</body>\n"

301

                            + "</html>");

302

});

303

        proxyRouter.route(HttpMethod.GET, "/style.css").handler(req -> {

304

            if (css == null) {

305

                css = getResourceAsString("style.css");

306

307

            req.response().end(css);

308

});

309

310

        // TODO This is no longer needed and can be removed at some point:

311

        proxyRouter.route(HttpMethod.GET, "/grlc-spec/*").handler(req -> {

312

            try {

313

                GrlcSpec gsp = new GrlcSpec(req.normalizedPath(), req.queryParams());

314

                req.response().putHeader("content-type", "text/yaml").end(gsp.getSpec());

315

            } catch (InvalidGrlcSpecException ex) {

316

                req.response().setStatusCode(400).end(ex.getMessage());

317

            } catch (Exception ex) {

318

                req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());

319

320

});

321

322

        proxyRouter.route(HttpMethod.GET, "/openapi/spec/*").handler(req -> {

323

            try {

324

                OpenApiSpecPage osp = new OpenApiSpecPage(req.normalizedPath(), req.queryParams());

325

                req.response().putHeader("content-type", "text/yaml").end(osp.getSpec());

326

            } catch (InvalidGrlcSpecException ex) {

327

                req.response().setStatusCode(400).end("Invlid grlc API definition: " + ex.getMessage());

328

            } catch (Exception ex) {

329

                req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());

330

331

});

332

333

        proxyRouter.route("/openapi/*").handler(StaticHandler.create("com/knowledgepixels/query/swagger"));

334

335

        HttpProxy grlcxProxy = HttpProxy.reverseProxy(httpClient);

336

        grlcxProxy.origin(proxyPort, proxy);

337

339

340

            @Override

341

            @GeneratedFlagForDependentElements

342

            public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {

343

                final ProxyRequest req = context.request();

344

                final String apiPattern = "^/api/(RA[a-zA-Z0-9-_]{43})/([a-zA-Z0-9-_]+)([?].*)?$";

345

                if (req.getURI().matches(apiPattern)) {

346

                    try {

347

                        GrlcSpec grlcSpec = new GrlcSpec(req.getURI(), req.proxiedRequest().params());

348

                        req.setMethod(HttpMethod.POST);

349

350

                        // Variant 1:

351

                        req.putHeader("Content-Type", "application/sparql-query");

352

                        req.setBody(Body.body(Buffer.buffer(grlcSpec.expandQuery())));

353

                        // Variant 2:

354

                        //req.putHeader("Content-Type", "application/x-www-form-urlencoded");

355

                        //req.setBody(Body.body(Buffer.buffer("query=" + URLEncoder.encode(grlcSpec.getExpandedQueryContent(), Charsets.UTF_8))));

356

357

                        req.setURI("/rdf4j-server/repositories/" + grlcSpec.getRepoName());

358

                        log.info("Forwarding apix request to /rdf4j-server/repositories/", grlcSpec.getRepoName());

359

                    } catch (InvalidGrlcSpecException ex) {

360

                        return Future.succeededFuture(context.request()

361

                                .response()

362

                                .setStatusCode(400)

363

                                .putHeader("Content-Type", "text/plain")

364

                                .setBody(Body.body(Buffer.buffer("Bad request: " + ex.getMessage()))));

365

                    } catch (Exception ex) {

366

                        return Future.succeededFuture(context.request()

367

                                .response()

368

                                .setStatusCode(500)

369

                                .putHeader("Content-Type", "text/plain")

370

                                .setBody(Body.body(Buffer.buffer("Unexpected error: " + ex.getMessage()))));

371

372

373

                return ProxyInterceptor.super.handleProxyRequest(context);

374

375

376

            @Override

377

            @GeneratedFlagForDependentElements

378

            public Future<Void> handleProxyResponse(ProxyContext context) {

379

                log.info("Receiving api response");

380

                ProxyResponse resp = context.response();

381

                resp.putHeader("Access-Control-Allow-Origin", "*");

382

                resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");

383

                resp.putHeader("Content-Disposition", "inline");

384

                return ProxyInterceptor.super.handleProxyResponse(context);

385

386

387

});

388

        proxyRouter.route(HttpMethod.GET, "/api/*").handler(ProxyHandler.create(grlcxProxy));

389

390

        proxyServer.requestHandler(req -> {

391

            applyGlobalHeaders(req.response());

392

            proxyRouter.handle(req);

393

});

394

        proxyServer.listen(9393);

395

396

        // Periodic metrics update

397

        vertx.setPeriodic(1000, id -> collector.updateMetrics());

398

399

400

        new Thread(() -> {

401

            try {

402

                var status = StatusController.get().initialize();

403

                log.info("Current state: {}, last committed counter: {}", status.state, status.loadCounter);

404

                if (status.state == StatusController.State.LAUNCHING || status.state == StatusController.State.LOADING_INITIAL) {

405

                    // Do the initial nanopublication loading

406

                    StatusController.get().setLoadingInitial(status.loadCounter);

407

                    // Fall back to local nanopub loading if the local files are present

408

                    if (!LocalNanopubLoader.init()) {

409

                        JellyNanopubLoader.loadInitial(status.loadCounter);

410

                    } else {

411

                        log.info("Local nanopublication loading finished");

412

413

                    StatusController.get().setReady();

414

                } else {

415

                    log.info("Initial load is already done");

416

                    StatusController.get().setReady();

417

418

            } catch (Exception ex) {

419

                log.info("Initial load failed, terminating...", ex);

420

                Runtime.getRuntime().exit(1);

421

422

423

            // Start periodic nanopub loading

424

            log.info("Starting periodic nanopub loading...");

425

            var executor = Executors.newSingleThreadScheduledExecutor();

426

            executor.scheduleWithFixedDelay(

427

                    JellyNanopubLoader::loadUpdates,

428

                    JellyNanopubLoader.UPDATES_POLL_INTERVAL,

429

                    JellyNanopubLoader.UPDATES_POLL_INTERVAL,

430

                    TimeUnit.MILLISECONDS

431

);

432

        }).start();

433

434

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {

435

            try {

436

                log.info("Gracefully shutting down...");

437

                TripleStore.get().shutdownRepositories();

438

                vertx.close().toCompletionStage().toCompletableFuture().get(5, TimeUnit.SECONDS);

439

                log.info("Graceful shutdown completed");

440

            } catch (Exception ex) {

441

                log.info("Graceful shutdown failed", ex);

442

443

        }));

444

445

446

    private String getResourceAsString(String file) {

447

        InputStream is = getClass().getClassLoader().getResourceAsStream("com/knowledgepixels/query/" + file);

448

        try (Scanner s = new Scanner(is).useDelimiter("\\A")) {

449

            String fileContent = s.hasNext() ? s.next() : "";

450

            return fileContent;

451

452

453

454

    private static void handleRedirect(RoutingContext req, String path) {

455

        String queryString = "";

456

        if (!req.queryParam("query").isEmpty())

457

            queryString = "?query=" + URLEncoder.encode(req.queryParam("query").get(0), Charsets.UTF_8);

458

        if (req.queryParam("for-type").size() == 1) {

459

            String type = req.queryParam("for-type").get(0);

460

            req.response().putHeader("location", path + "/type/" + Utils.createHash(type) + queryString);

461

            req.response().setStatusCode(301).end();

462

        } else if (req.queryParam("for-pubkey").size() == 1) {

463

            String type = req.queryParam("for-pubkey").get(0);

464

            req.response().putHeader("location", path + "/pubkey/" + Utils.createHash(type) + queryString);

465

            req.response().setStatusCode(301).end();

466

        } else if (req.queryParam("for-user").size() == 1) {

467

            String type = req.queryParam("for-user").get(0);

468

            req.response().putHeader("location", path + "/user/" + Utils.createHash(type) + queryString);

469

            req.response().setStatusCode(301).end();

470

471

472

473

/**

474

     * Apply headers to the response that should be present for all requests.

475

476

     * @param response The response to which the headers should be applied.

477

*/

478

    private static void applyGlobalHeaders(HttpServerResponse response) {

479

        response.putHeader("Nanopub-Query-Status", StatusController.get().getState().state.toString());

480

481

knowledgepixels / nanopub-query / 17945178198

Source File
Press 'n' to go to next uncovered line, 'b' for previous

1	package com.knowledgepixels.query;
2
3	import java.io.InputStream;
4	import java.net.URLEncoder;
5	import java.util.ArrayList;
6	import java.util.Collections;
7	import java.util.List;
8	import java.util.Scanner;
9	import java.util.concurrent.Executors;
10	import java.util.concurrent.TimeUnit;
11
12	import org.eclipse.rdf4j.model.Value;
13	import org.slf4j.Logger;
14	import org.slf4j.LoggerFactory;
15
16	import com.github.jsonldjava.shaded.com.google.common.base.Charsets;
17	import com.knowledgepixels.query.GrlcSpec.InvalidGrlcSpecException;
18
19	import io.micrometer.prometheus.PrometheusMeterRegistry;
20	import io.vertx.core.AbstractVerticle;
21	import io.vertx.core.Future;
22	import io.vertx.core.Promise;
23	import io.vertx.core.buffer.Buffer;
24	import io.vertx.core.http.HttpClient;
25	import io.vertx.core.http.HttpClientOptions;
26	import io.vertx.core.http.HttpMethod;
27	import io.vertx.core.http.HttpServer;
28	import io.vertx.core.http.HttpServerResponse;
29	import io.vertx.core.http.PoolOptions;
30	import io.vertx.ext.web.Router;
31	import io.vertx.ext.web.RoutingContext;
32	import io.vertx.ext.web.handler.CorsHandler;
33	import io.vertx.ext.web.handler.StaticHandler;
34	import io.vertx.ext.web.proxy.handler.ProxyHandler;
35	import io.vertx.httpproxy.Body;
36	import io.vertx.httpproxy.HttpProxy;
37	import io.vertx.httpproxy.ProxyContext;
38	import io.vertx.httpproxy.ProxyInterceptor;
39	import io.vertx.httpproxy.ProxyRequest;
40	import io.vertx.httpproxy.ProxyResponse;
41	import io.vertx.micrometer.PrometheusScrapingHandler;
42	import io.vertx.micrometer.backends.BackendRegistries;
43
44	/**
45	* Main verticle that coordinates the incoming HTTP requests.
46	*/
47	@GeneratedFlagForDependentElements
48	public class MainVerticle extends AbstractVerticle {
49
50	private static String css = null;
51
52	private static final Logger log = LoggerFactory.getLogger(MainVerticle.class);
53
54	/**
55	* Start the main verticle.
56	*
57	* @param startPromise the promise to complete when the verticle is started
58	* @throws Exception if an error occurs during startup
59	*/
60	@Override
61	public void start(Promise<Void> startPromise) throws Exception {
62	HttpClient httpClient = vertx.createHttpClient(
63	new HttpClientOptions()
64	.setConnectTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_CONNECT_TIMEOUT", 1000))
65	.setIdleTimeoutUnit(TimeUnit.SECONDS)
66	.setIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))
67	.setReadIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))
68	.setWriteIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60)),
69	new PoolOptions().setHttp1MaxSize(200).setHttp2MaxSize(200)
70	);
71
72	HttpServer proxyServer = vertx.createHttpServer();
73	Router proxyRouter = Router.router(vertx);
74	proxyRouter.route().handler(CorsHandler.create().addRelativeOrigin(".*"));
75
76	// Metrics
77	final var metricsHttpServer = vertx.createHttpServer();
78	final var metricsRouter = Router.router(vertx);
79	metricsHttpServer.requestHandler(metricsRouter).listen(9394);
80
81	final var metricsRegistry = (PrometheusMeterRegistry) BackendRegistries.getDefaultNow();
82	final var collector = new MetricsCollector(metricsRegistry);
83	metricsRouter.route("/metrics").handler(PrometheusScrapingHandler.create(metricsRegistry));
84	// ----------
85	// This part is only used if the redirection is not done through Nginx.
86	// See nginx.conf and this bug report: https://github.com/eclipse-rdf4j/rdf4j/discussions/5120
87	HttpProxy rdf4jProxy = HttpProxy.reverseProxy(httpClient);
88	String proxy = Utils.getEnvString("RDF4J_PROXY_HOST", "rdf4j");
89	int proxyPort = Utils.getEnvInt("RDF4J_PROXY_PORT", 8080);
90	rdf4jProxy.origin(proxyPort, proxy);
91
92	rdf4jProxy.addInterceptor(new ProxyInterceptor() {	×
93
94	@Override
95	@GeneratedFlagForDependentElements
96	public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {
97	ProxyRequest request = context.request();
98	request.setURI(request.getURI().replaceAll("/", "_").replaceFirst("^_repo_", "/rdf4j-server/repositories/"));
99	// For later to try to get HTML tables out:
100	// if (request.headers().get("Accept") == null) {
101	// request.putHeader("Accept", "text/html");
102	// }
103	// request.putHeader("Accept", "application/json");
104	return ProxyInterceptor.super.handleProxyRequest(context);
105	}
106
107	@Override
108	@GeneratedFlagForDependentElements
109	public Future<Void> handleProxyResponse(ProxyContext context) {
110	ProxyResponse resp = context.response();
111	resp.putHeader("Access-Control-Allow-Origin", "*");
112	resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");
113	// For later to try to get HTML tables out:
114	// String acceptHeader = context.request().headers().get("Accept");
115	// if (acceptHeader != null && acceptHeader.contains("text/html")) {
116	// resp.putHeader("Content-Type", "text/html");
117	// resp.setBody(Body.body(Buffer.buffer("<html><body><strong>test</strong></body></html>")));
118	// }
119	return ProxyInterceptor.super.handleProxyResponse(context);
120	}
121
122	});
123	// ----------
124
125	proxyRouter.route(HttpMethod.GET, "/repo").handler(req -> handleRedirect(req, "/repo"));
126	proxyRouter.route(HttpMethod.GET, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));
127	proxyRouter.route(HttpMethod.POST, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));
128	proxyRouter.route(HttpMethod.HEAD, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));
129	proxyRouter.route(HttpMethod.OPTIONS, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));
130	proxyRouter.route(HttpMethod.GET, "/tools/*").handler(req -> {
131	final String yasguiPattern = "^/tools/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?/yasgui\\.html$";
132	if (req.normalizedPath().matches(yasguiPattern)) {
133	String repo = req.normalizedPath().replaceFirst(yasguiPattern, "$1$2");
134	req.response()
135	.putHeader("content-type", "text/html")
136	.end("<!DOCTYPE html>\n"
137	+ "<html lang=\"en\">\n"
138	+ "<head>\n"
139	+ "<meta charset=\"utf-8\">\n"
140	+ "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"
141	+ "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
142	+ "<title>Nanopub Query SPARQL Editor for repository: " + repo + "</title>\n"
143	+ "<link rel=\"stylesheet\" href=\"/style.css\">\n"
144	+ "<link href='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.css' rel='stylesheet' type='text/css'/>\n"
145	+ "<style>.yasgui .endpointText {display:none !important;}</style>\n"
146	+ "<script type=\"text/javascript\">localStorage.clear();</script>\n"
147	+ "</head>\n"
148	+ "<body>\n"
149	+ "<h3>Nanopub Query SPARQL Editor for repository: " + repo + "</h3>\n"
150	+ "<div id='yasgui'></div>\n"
151	+ "<script src='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.js'></script>\n"
152	+ "<script type=\"text/javascript\">\n"
153	+ "var yasgui = YASGUI(document.getElementById(\"yasgui\"), {\n"
154	+ " yasqe:{sparql:{endpoint:'/repo/" + repo + "'},value:'" + Utils.defaultQuery.replaceAll("\n", "\\\\n") + "'}\n"
155	+ "});\n"
156	+ "</script>\n"
157	+ "</body>\n"
158	+ "</html>");
159	} else {
160	req.response()
161	.putHeader("content-type", "text/plain")
162	.setStatusCode(404)
163	.end("not found");
164	}
165	});
166	proxyRouter.route(HttpMethod.GET, "/page").handler(req -> handleRedirect(req, "/page"));
167	proxyRouter.route(HttpMethod.GET, "/page/*").handler(req -> {
168	final String pagePattern = "^/page/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?$";
169	if (req.normalizedPath().matches(pagePattern)) {
170	String repo = req.normalizedPath().replaceFirst(pagePattern, "$1$2");
171	req.response()
172	.putHeader("content-type", "text/html")
173	.end("<!DOCTYPE html>\n"
174	+ "<html lang=\"en\">\n"
175	+ "<head>\n"
176	+ "<meta charset=\"utf-8\">\n"
177	+ "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"
178	+ "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"
179	+ "<title>Nanopub Query repo: " + repo + "</title>\n"
180	+ "<link rel=\"stylesheet\" href=\"/style.css\">\n"
181	+ "</head>\n"
182	+ "<body>\n"
183	+ "<h3>Nanopub Query repo: " + repo + "</h3>\n"
184	+ "<p>Endpoint: <a href=\"/repo/" + repo + "\">/repo/" + repo + "</a></p>"
185	+ "<p>YASGUI: <a href=\"/tools/" + repo + "/yasgui.html\">/tools/" + repo + "/yasgui.hml</a></p>"
186	+ "</body>\n"
187	+ "</html>");
188	} else {
189	req.response()
190	.putHeader("content-type", "text/plain")
191	.setStatusCode(404)
192	.end("not found");
193	}
194	});
195	proxyRouter.route(HttpMethod.GET, "/").handler(req -> {
196	String repos = "";
197	List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());
198	Collections.sort(repoList);
199	for (String s : repoList) {
200	if (s.startsWith("pubkey_") \|\| s.startsWith("type_")) continue;
201	repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a></code></li>";
202	}
203	String pinnedApisValue = Utils.getEnvString("NANOPUB_QUERY_PINNED_APIS", "");
204	String[] pinnedApis = pinnedApisValue.split(" ");
205	String pinnedApiLinks = "";
206	if (!pinnedApisValue.isEmpty()) {
207	for (String s : pinnedApis) {
208	pinnedApiLinks = pinnedApiLinks + "<li><a href=\"openapi/?url=spec/" + s + "%3Fapi-version=latest\">" + s.replaceFirst("^.*/", "") + "</a></li>";
209	}
210	pinnedApiLinks = "<p>Pinned APIs:</p>\n" +
211	"<ul>\n" +
212	pinnedApiLinks +
213	"</ul>\n";
214	}
215	req.response()
216	.putHeader("content-type", "text/html")
217	.end("<!DOCTYPE html>\n"
218	+ "<html lang='en'>\n"
219	+ "<head>\n"
220	+ "<title>Nanopub Query</title>\n"
221	+ "<meta charset='utf-8'>\n"
222	+ "<link rel=\"stylesheet\" href=\"/style.css\">\n"
223	+ "</head>\n"
224	+ "<body>\n"
225	+ "<h1>Nanopub Query</h1>"
226	+ "<p>General repos:</p>"
227	+ "<ul>" + repos + "</ul>"
228	+ "<p>Specific repos:</p>"
229	+ "<ul>"
230	+ "<li><a href=\"/pubkeys\">Pubkey Repos</a></li>"
231	+ "<li><a href=\"/types\">Type Repos</a></li>"
232	+ "</ul>"
233	+ pinnedApiLinks
234	+ "</body>\n"
235	+ "</html>");
236	});
237	proxyRouter.route(HttpMethod.GET, "/pubkeys").handler(req -> {
238	String repos = "";
239	List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());
240	Collections.sort(repoList);
241	for (String s : repoList) {
242	if (!s.startsWith("pubkey_")) continue;
243	String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");
244	Value hashObj = Utils.getObjectForHash(hash);
245	String label;
246	if (hashObj == null) {
247	label = "";
248	} else {
249	label = " (" + Utils.getShortPubkeyName(hashObj.stringValue()) + ")";
250	}
251	s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");
252	repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";
253	}
254	req.response()
255	.putHeader("content-type", "text/html")
256	.end("<!DOCTYPE html>\n"
257	+ "<html lang='en'>\n"
258	+ "<head>\n"
259	+ "<title>Nanopub Query: Pubkey Repos</title>\n"
260	+ "<meta charset='utf-8'>\n"
261	+ "<link rel=\"stylesheet\" href=\"/style.css\">\n"
262	+ "</head>\n"
263	+ "<body>\n"
264	+ "<h3>Pubkey Repos</h3>"
265	+ "<p>Repos:</p>"
266	+ "<ul>" + repos + "</ul>"
267	+ "</body>\n"
268	+ "</html>");
269	});
270	proxyRouter.route(HttpMethod.GET, "/types").handler(req -> {
271	String repos = "";
272	List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());
273	Collections.sort(repoList);
274	for (String s : repoList) {
275	if (!s.startsWith("type_")) continue;
276	String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");
277	Value hashObj = Utils.getObjectForHash(hash);
278	String label;
279	if (hashObj == null) {
280	label = "";
281	} else {
282	label = " (" + hashObj.stringValue() + ")";
283	}
284	s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");
285	repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";
286	}
287	req.response()
288	.putHeader("content-type", "text/html")
289	.end("<!DOCTYPE html>\n"
290	+ "<html lang='en'>\n"
291	+ "<head>\n"
292	+ "<title>Nanopub Query: Type Repos</title>\n"
293	+ "<meta charset='utf-8'>\n"
294	+ "<link rel=\"stylesheet\" href=\"/style.css\">\n"
295	+ "</head>\n"
296	+ "<body>\n"
297	+ "<h3>Type Repos</h3>"
298	+ "<p>Repos:</p>"
299	+ "<ul>" + repos + "</ul>"
300	+ "</body>\n"
301	+ "</html>");
302	});
303	proxyRouter.route(HttpMethod.GET, "/style.css").handler(req -> {
304	if (css == null) {
305	css = getResourceAsString("style.css");
306	}
307	req.response().end(css);
308	});
309
310	// TODO This is no longer needed and can be removed at some point:
311	proxyRouter.route(HttpMethod.GET, "/grlc-spec/*").handler(req -> {
312	try {
313	GrlcSpec gsp = new GrlcSpec(req.normalizedPath(), req.queryParams());
314	req.response().putHeader("content-type", "text/yaml").end(gsp.getSpec());
315	} catch (InvalidGrlcSpecException ex) {
316	req.response().setStatusCode(400).end(ex.getMessage());
317	} catch (Exception ex) {
318	req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());
319	}
320	});
321
322	proxyRouter.route(HttpMethod.GET, "/openapi/spec/*").handler(req -> {
323	try {
324	OpenApiSpecPage osp = new OpenApiSpecPage(req.normalizedPath(), req.queryParams());
325	req.response().putHeader("content-type", "text/yaml").end(osp.getSpec());
326	} catch (InvalidGrlcSpecException ex) {
327	req.response().setStatusCode(400).end("Invlid grlc API definition: " + ex.getMessage());
328	} catch (Exception ex) {
329	req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());
330	}
331	});
332
333	proxyRouter.route("/openapi/*").handler(StaticHandler.create("com/knowledgepixels/query/swagger"));
334
335	HttpProxy grlcxProxy = HttpProxy.reverseProxy(httpClient);
336	grlcxProxy.origin(proxyPort, proxy);
337
338	grlcxProxy.addInterceptor(new ProxyInterceptor() {	×
339
340	@Override
341	@GeneratedFlagForDependentElements
342	public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {
343	final ProxyRequest req = context.request();
344	final String apiPattern = "^/api/(RA[a-zA-Z0-9-_]{43})/([a-zA-Z0-9-_]+)([?].*)?$";
345	if (req.getURI().matches(apiPattern)) {
346	try {
347	GrlcSpec grlcSpec = new GrlcSpec(req.getURI(), req.proxiedRequest().params());
348	req.setMethod(HttpMethod.POST);
349
350	// Variant 1:
351	req.putHeader("Content-Type", "application/sparql-query");
352	req.setBody(Body.body(Buffer.buffer(grlcSpec.expandQuery())));
353	// Variant 2:
354	//req.putHeader("Content-Type", "application/x-www-form-urlencoded");
355	//req.setBody(Body.body(Buffer.buffer("query=" + URLEncoder.encode(grlcSpec.getExpandedQueryContent(), Charsets.UTF_8))));
356
357	req.setURI("/rdf4j-server/repositories/" + grlcSpec.getRepoName());
358	log.info("Forwarding apix request to /rdf4j-server/repositories/", grlcSpec.getRepoName());
359	} catch (InvalidGrlcSpecException ex) {
360	return Future.succeededFuture(context.request()
361	.response()
362	.setStatusCode(400)
363	.putHeader("Content-Type", "text/plain")
364	.setBody(Body.body(Buffer.buffer("Bad request: " + ex.getMessage()))));
365	} catch (Exception ex) {
366	return Future.succeededFuture(context.request()
367	.response()
368	.setStatusCode(500)
369	.putHeader("Content-Type", "text/plain")
370	.setBody(Body.body(Buffer.buffer("Unexpected error: " + ex.getMessage()))));
371	}
372	}
373	return ProxyInterceptor.super.handleProxyRequest(context);
374	}
375
376	@Override
377	@GeneratedFlagForDependentElements
378	public Future<Void> handleProxyResponse(ProxyContext context) {
379	log.info("Receiving api response");
380	ProxyResponse resp = context.response();
381	resp.putHeader("Access-Control-Allow-Origin", "*");
382	resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");
383	resp.putHeader("Content-Disposition", "inline");
384	return ProxyInterceptor.super.handleProxyResponse(context);
385	}
386
387	});
388	proxyRouter.route(HttpMethod.GET, "/api/*").handler(ProxyHandler.create(grlcxProxy));
389
390	proxyServer.requestHandler(req -> {
391	applyGlobalHeaders(req.response());
392	proxyRouter.handle(req);
393	});
394	proxyServer.listen(9393);
395
396	// Periodic metrics update
397	vertx.setPeriodic(1000, id -> collector.updateMetrics());
398
399
400	new Thread(() -> {
401	try {
402	var status = StatusController.get().initialize();
403	log.info("Current state: {}, last committed counter: {}", status.state, status.loadCounter);
404	if (status.state == StatusController.State.LAUNCHING \|\| status.state == StatusController.State.LOADING_INITIAL) {
405	// Do the initial nanopublication loading
406	StatusController.get().setLoadingInitial(status.loadCounter);
407	// Fall back to local nanopub loading if the local files are present
408	if (!LocalNanopubLoader.init()) {
409	JellyNanopubLoader.loadInitial(status.loadCounter);
410	} else {
411	log.info("Local nanopublication loading finished");
412	}
413	StatusController.get().setReady();
414	} else {
415	log.info("Initial load is already done");
416	StatusController.get().setReady();
417	}
418	} catch (Exception ex) {
419	log.info("Initial load failed, terminating...", ex);
420	Runtime.getRuntime().exit(1);
421	}
422
423	// Start periodic nanopub loading
424	log.info("Starting periodic nanopub loading...");
425	var executor = Executors.newSingleThreadScheduledExecutor();
426	executor.scheduleWithFixedDelay(
427	JellyNanopubLoader::loadUpdates,
428	JellyNanopubLoader.UPDATES_POLL_INTERVAL,
429	JellyNanopubLoader.UPDATES_POLL_INTERVAL,
430	TimeUnit.MILLISECONDS
431	);
432	}).start();
433
434	Runtime.getRuntime().addShutdownHook(new Thread(() -> {
435	try {
436	log.info("Gracefully shutting down...");
437	TripleStore.get().shutdownRepositories();
438	vertx.close().toCompletionStage().toCompletableFuture().get(5, TimeUnit.SECONDS);
439	log.info("Graceful shutdown completed");
440	} catch (Exception ex) {
441	log.info("Graceful shutdown failed", ex);
442	}
443	}));
444	}
445
446	private String getResourceAsString(String file) {
447	InputStream is = getClass().getClassLoader().getResourceAsStream("com/knowledgepixels/query/" + file);
448	try (Scanner s = new Scanner(is).useDelimiter("\\A")) {
449	String fileContent = s.hasNext() ? s.next() : "";
450	return fileContent;
451	}
452	}
453
454	private static void handleRedirect(RoutingContext req, String path) {
455	String queryString = "";
456	if (!req.queryParam("query").isEmpty())
457	queryString = "?query=" + URLEncoder.encode(req.queryParam("query").get(0), Charsets.UTF_8);
458	if (req.queryParam("for-type").size() == 1) {
459	String type = req.queryParam("for-type").get(0);
460	req.response().putHeader("location", path + "/type/" + Utils.createHash(type) + queryString);
461	req.response().setStatusCode(301).end();
462	} else if (req.queryParam("for-pubkey").size() == 1) {
463	String type = req.queryParam("for-pubkey").get(0);
464	req.response().putHeader("location", path + "/pubkey/" + Utils.createHash(type) + queryString);
465	req.response().setStatusCode(301).end();
466	} else if (req.queryParam("for-user").size() == 1) {
467	String type = req.queryParam("for-user").get(0);
468	req.response().putHeader("location", path + "/user/" + Utils.createHash(type) + queryString);
469	req.response().setStatusCode(301).end();
470	}
471	}
472
473	/**
474	* Apply headers to the response that should be present for all requests.
475	*
476	* @param response The response to which the headers should be applied.
477	*/
478	private static void applyGlobalHeaders(HttpServerResponse response) {
479	response.putHeader("Nanopub-Query-Status", StatusController.get().getState().state.toString());
480	}
481	}

knowledgepixels / nanopub-query / 17945178198

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous