knowledgepixels/nanopub-query | Build 30254746745 | com/knowledgepixels/query/MainVerticle.java | Coveralls

1

package com.knowledgepixels.query;

2

3

import com.github.jsonldjava.shaded.com.google.common.base.Charsets;

4

import com.knowledgepixels.query.GrlcSpec.InvalidGrlcSpecException;

5

import io.micrometer.prometheus.PrometheusMeterRegistry;

6

import io.vertx.core.AbstractVerticle;

7

import io.vertx.core.Future;

8

import io.vertx.core.Promise;

9

import io.vertx.core.buffer.Buffer;

10

import io.vertx.core.http.*;

11

import io.vertx.ext.web.Router;

12

import io.vertx.ext.web.RoutingContext;

13

import io.vertx.ext.web.handler.CorsHandler;

14

import io.vertx.ext.web.handler.StaticHandler;

15

import io.vertx.ext.web.proxy.handler.ProxyHandler;

16

import io.vertx.httpproxy.*;

17

import io.vertx.micrometer.PrometheusScrapingHandler;

18

import io.vertx.micrometer.backends.BackendRegistries;

19

import org.eclipse.rdf4j.model.Value;

20

import org.slf4j.Logger;

21

import org.slf4j.LoggerFactory;

22

23

import java.io.InputStream;

24

import java.net.URLEncoder;

25

import java.util.ArrayList;

26

import java.util.Collections;

27

import java.util.List;

28

import java.util.Scanner;

29

import java.util.concurrent.Executors;

30

import java.util.concurrent.TimeUnit;

31

32

/**

33

 * Main verticle that coordinates the incoming HTTP requests.

34

*/

35

@GeneratedFlagForDependentElements

36

public class MainVerticle extends AbstractVerticle {

37

38

    private static String css = null;

39

40

    private static final Logger logger = LoggerFactory.getLogger(MainVerticle.class);

41

42

/**

43

     * Start the main verticle.

44

45

     * @param startPromise the promise to complete when the verticle is started

46

     * @throws Exception if an error occurs during startup

47

*/

48

    @Override

49

    public void start(Promise<Void> startPromise) throws Exception {

50

        if (!FeatureFlags.trustStateEnabled()) {

51

            logger.warn("Trust state feature disabled via NANOPUB_QUERY_ENABLE_TRUST_STATE=false — "

52

                    + "no trust snapshots will be fetched or materialised, and the 'trust' repo will not be auto-created.");

53

54

        if (!FeatureFlags.spacesEnabled()) {

55

            logger.warn("Spaces feature disabled via NANOPUB_QUERY_ENABLE_SPACES=false — "

56

                    + "no space-relevant nanopubs will be extracted into npa:spacesGraph, "

57

                    + "and the 'spaces' repo will not be auto-created.");

58

59

        if (!FeatureFlags.fullRepoEnabled()) {

60

            logger.warn("Writes to the 'full' repo disabled via NANOPUB_QUERY_ENABLE_FULL_REPO=false — "

61

                    + "generic SPARQL queries against /repo/full will return an empty store.");

62

63

        if (!FeatureFlags.textRepoEnabled()) {

64

            logger.warn("Writes to the 'text' repo disabled via NANOPUB_QUERY_ENABLE_TEXT_REPO=false — "

65

                    + "full-text search via /repo/text will return nothing.");

66

67

        if (!FeatureFlags.last30dRepoEnabled()) {

68

            logger.warn("Writes to the 'last30d' repo disabled via NANOPUB_QUERY_ENABLE_LAST30D_REPO=false — "

69

                    + "the /repo/last30d endpoint will be empty; rewrite queries against /repo/full with a date filter.");

70

71

        if (!FeatureFlags.reconciliationEnabled()) {

72

            logger.warn("Shard reconciliation disabled via NANOPUB_QUERY_ENABLE_RECONCILIATION=false — "

73

                    + "nanopubs silently missing from individual shard repos (issue #139) will not be detected or repaired.");

74

75

        HttpClient httpClient = vertx.createHttpClient(

76

                new HttpClientOptions()

77

                        .setConnectTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_CONNECT_TIMEOUT", 1000))

78

                        .setIdleTimeoutUnit(TimeUnit.SECONDS)

79

                        .setIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))

80

                        .setReadIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60))

81

                        .setWriteIdleTimeout(Utils.getEnvInt("NANOPUB_QUERY_VERTX_IDLE_TIMEOUT", 60)),

82

                new PoolOptions().setHttp1MaxSize(200).setHttp2MaxSize(200)

83

);

84

85

        HttpServer proxyServer = vertx.createHttpServer(

86

                new HttpServerOptions().setMaxInitialLineLength(65536)

87

);

88

        Router proxyRouter = Router.router(vertx);

89

        proxyRouter.route().handler(CorsHandler.create().addRelativeOrigin(".*"));

90

91

        // Metrics

92

        final var metricsHttpServer = vertx.createHttpServer();

93

        final var metricsRouter = Router.router(vertx);

94

        metricsHttpServer.requestHandler(metricsRouter).listen(9394);

95

96

        final var metricsRegistry = (PrometheusMeterRegistry) BackendRegistries.getDefaultNow();

97

        final var collector = new MetricsCollector(metricsRegistry);

98

        metricsRouter.route("/metrics").handler(PrometheusScrapingHandler.create(metricsRegistry));

99

        // ----------

100

        // This part is only used if the redirection is not done through Nginx.

101

        // See nginx.conf and this bug report: https://github.com/eclipse-rdf4j/rdf4j/discussions/5120

102

        HttpProxy rdf4jProxy = HttpProxy.reverseProxy(httpClient);

103

        String proxy = Utils.getEnvString("RDF4J_PROXY_HOST", "rdf4j");

104

        int proxyPort = Utils.getEnvInt("RDF4J_PROXY_PORT", 8080);

105

        rdf4jProxy.origin(proxyPort, proxy);

106

108

109

            @Override

110

            @GeneratedFlagForDependentElements

111

            public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {

112

                ProxyRequest request = context.request();

113

                request.setURI(request.getURI().replaceAll("/", "_").replaceFirst("^_repo_", "/rdf4j-server/repositories/"));

114

                // For later to try to get HTML tables out:

115

//                                if (request.headers().get("Accept") == null) {

116

//                                        request.putHeader("Accept", "text/html");

117

//                                }

118

//                                request.putHeader("Accept", "application/json");

119

                return ProxyInterceptor.super.handleProxyRequest(context);

120

121

122

            @Override

123

            @GeneratedFlagForDependentElements

124

            public Future<Void> handleProxyResponse(ProxyContext context) {

125

                ProxyResponse resp = context.response();

126

                resp.putHeader("Access-Control-Allow-Origin", "*");

127

                resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");

128

                // For later to try to get HTML tables out:

129

//                                String acceptHeader = context.request().headers().get("Accept");

130

//                                if (acceptHeader != null && acceptHeader.contains("text/html")) {

131

//                                        resp.putHeader("Content-Type", "text/html");

132

//                                        resp.setBody(Body.body(Buffer.buffer("<html><body><strong>test</strong></body></html>")));

133

//                                }

134

                return ProxyInterceptor.super.handleProxyResponse(context);

135

136

137

});

138

        // ----------

139

140

        proxyRouter.route(HttpMethod.GET, "/repo").handler(req -> handleRedirect(req, "/repo"));

141

        proxyRouter.route(HttpMethod.GET, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

142

        proxyRouter.route(HttpMethod.POST, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

143

        proxyRouter.route(HttpMethod.HEAD, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

144

        proxyRouter.route(HttpMethod.OPTIONS, "/repo/*").handler(ProxyHandler.create(rdf4jProxy));

145

        proxyRouter.route(HttpMethod.GET, "/tools/*").handler(req -> {

146

            final String yasguiPattern = "^/tools/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?/yasgui\\.html$";

147

            if (req.normalizedPath().matches(yasguiPattern)) {

148

                String repo = req.normalizedPath().replaceFirst(yasguiPattern, "$1$2");

149

                req.response()

150

                        .putHeader("content-type", "text/html")

151

                        .end("<!DOCTYPE html>\n"

152

                             + "<html lang=\"en\">\n"

153

                             + "<head>\n"

154

                             + "<meta charset=\"utf-8\">\n"

155

                             + "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"

156

                             + "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"

157

                             + "<title>Nanopub Query SPARQL Editor for repository: " + repo + "</title>\n"

158

                             + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

159

                             + "<link href='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.css' rel='stylesheet' type='text/css'/>\n"

160

                             + "<style>.yasgui .endpointText {display:none !important;}</style>\n"

161

                             + "<script type=\"text/javascript\">localStorage.clear();</script>\n"

162

                             + "</head>\n"

163

                             + "<body>\n"

164

                             + "<h3>Nanopub Query SPARQL Editor for repository: " + repo + "</h3>\n"

165

                             + "<div id='yasgui'></div>\n"

166

                             + "<script src='https://cdn.jsdelivr.net/yasgui/2.6.1/yasgui.min.js'></script>\n"

167

                             + "<script type=\"text/javascript\">\n"

168

                             + "var yasgui = YASGUI(document.getElementById(\"yasgui\"), {\n"

169

                             + "  yasqe:{sparql:{endpoint:'/repo/" + repo + "'},value:'" + Utils.defaultQuery.replaceAll("\n", "\\\\n") + "'}\n"

170

                             + "});\n"

171

                             + "</script>\n"

172

                             + "</body>\n"

173

                             + "</html>");

174

            } else {

175

                req.response()

176

                        .putHeader("content-type", "text/plain")

177

                        .setStatusCode(404)

178

                        .end("not found");

179

180

});

181

        proxyRouter.route(HttpMethod.GET, "/page").handler(req -> handleRedirect(req, "/page"));

182

        proxyRouter.route(HttpMethod.GET, "/page/*").handler(req -> {

183

            final String pagePattern = "^/page/([a-zA-Z0-9-_]+)(/([a-zA-Z0-9-_]+))?$";

184

            if (req.normalizedPath().matches(pagePattern)) {

185

                String repo = req.normalizedPath().replaceFirst(pagePattern, "$1$2");

186

                req.response()

187

                        .putHeader("content-type", "text/html")

188

                        .end("<!DOCTYPE html>\n"

189

                             + "<html lang=\"en\">\n"

190

                             + "<head>\n"

191

                             + "<meta charset=\"utf-8\">\n"

192

                             + "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">\n"

193

                             + "<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n"

194

                             + "<title>Nanopub Query repo: " + repo + "</title>\n"

195

                             + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

196

                             + "</head>\n"

197

                             + "<body>\n"

198

                             + "<h3>Nanopub Query repo: " + repo + "</h3>\n"

199

                             + "<p>Endpoint: <a href=\"/repo/" + repo + "\">/repo/" + repo + "</a></p>"

200

                             + "<p>YASGUI: <a href=\"/tools/" + repo + "/yasgui.html\">/tools/" + repo + "/yasgui.hml</a></p>"

201

                             + "</body>\n"

202

                             + "</html>");

203

            } else {

204

                req.response()

205

                        .putHeader("content-type", "text/plain")

206

                        .setStatusCode(404)

207

                        .end("not found");

208

209

});

210

        proxyRouter.route(HttpMethod.GET, "/").handler(req -> {

211

            vertx.<String>executeBlocking(() -> {

212

                String repos = "";

213

                List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

214

                Collections.sort(repoList);

215

                for (String s : repoList) {

216

                    if (s.startsWith("pubkey_") || s.startsWith("type_")) continue;

217

                    repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a></code></li>";

218

219

                String pinnedApisValue = Utils.getEnvString("NANOPUB_QUERY_PINNED_APIS", "");

220

                String[] pinnedApis = pinnedApisValue.split(" ");

221

                String pinnedApiLinks = "";

222

                if (!pinnedApisValue.isEmpty()) {

223

                    for (String s : pinnedApis) {

224

                        pinnedApiLinks = pinnedApiLinks + "<li><a href=\"openapi/?url=spec/" + s + "%3Fapi-version=latest\">" + s.replaceFirst("^.*/", "") + "</a></li>";

225

226

                    pinnedApiLinks = "<p>Pinned APIs:</p>\n" +

227

                                     "<ul>\n" +

228

                                     pinnedApiLinks +

229

                                     "</ul>\n";

230

231

                return "<!DOCTYPE html>\n"

232

                     + "<html lang='en'>\n"

233

                     + "<head>\n"

234

                     + "<title>Nanopub Query</title>\n"

235

                     + "<meta charset='utf-8'>\n"

236

                     + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

237

                     + "</head>\n"

238

                     + "<body>\n"

239

                     + "<h1>Nanopub Query</h1>"

240

                     + "<p>General repos:</p>"

241

                     + "<ul>" + repos + "</ul>"

242

                     + "<p>Specific repos:</p>"

243

                     + "<ul>"

244

                     + "<li><a href=\"/pubkeys\">Pubkey Repos</a></li>"

245

                     + "<li><a href=\"/types\">Type Repos</a></li>"

246

                     + "</ul>"

247

                     + (FeatureFlags.spacesEnabled()

248

                             ? "<p>Spaces:</p>"

249

                               + "<ul><li><a href=\"/spaces\">Spaces</a></li></ul>"

250

                             : "")

251

                     + pinnedApiLinks

252

                     + "</body>\n"

253

                     + "</html>";

254

            }, false).onSuccess(html -> {

255

                req.response().putHeader("content-type", "text/html").end(html);

256

            }).onFailure(ex -> {

257

                req.response().setStatusCode(500).end("Error: " + ex.getMessage());

258

});

259

});

260

        proxyRouter.route(HttpMethod.GET, "/pubkeys").handler(req -> {

261

            vertx.<String>executeBlocking(() -> {

262

                String repos = "";

263

                List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

264

                Collections.sort(repoList);

265

                for (String s : repoList) {

266

                    if (!s.startsWith("pubkey_")) continue;

267

                    String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");

268

                    Value hashObj = Utils.getObjectForHash(hash);

269

                    String label;

270

                    if (hashObj == null) {

271

                        label = "";

272

                    } else {

273

                        label = " (" + Utils.getShortPubkeyName(hashObj.stringValue()) + ")";

274

275

                    s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");

276

                    repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";

277

278

                return "<!DOCTYPE html>\n"

279

                     + "<html lang='en'>\n"

280

                     + "<head>\n"

281

                     + "<title>Nanopub Query: Pubkey Repos</title>\n"

282

                     + "<meta charset='utf-8'>\n"

283

                     + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

284

                     + "</head>\n"

285

                     + "<body>\n"

286

                     + "<h3>Pubkey Repos</h3>"

287

                     + "<p>Repos:</p>"

288

                     + "<ul>" + repos + "</ul>"

289

                     + "</body>\n"

290

                     + "</html>";

291

            }, false).onSuccess(html -> {

292

                req.response().putHeader("content-type", "text/html").end(html);

293

            }).onFailure(ex -> {

294

                req.response().setStatusCode(500).end("Error: " + ex.getMessage());

295

});

296

});

297

        proxyRouter.route(HttpMethod.GET, "/types").handler(req -> {

298

            vertx.<String>executeBlocking(() -> {

299

                String repos = "";

300

                List<String> repoList = new ArrayList<>(TripleStore.get().getRepositoryNames());

301

                Collections.sort(repoList);

302

                for (String s : repoList) {

303

                    if (!s.startsWith("type_")) continue;

304

                    String hash = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$2");

305

                    Value hashObj = Utils.getObjectForHash(hash);

306

                    String label;

307

                    if (hashObj == null) {

308

                        label = "";

309

                    } else {

310

                        label = " (" + hashObj.stringValue() + ")";

311

312

                    s = s.replaceFirst("^([a-zA-Z0-9-]+)_([a-zA-Z0-9-_]+)$", "$1/$2");

313

                    repos += "<li><code><a href=\"/page/" + s + "\">" + s + "</a>" + label + "</code></li>";

314

315

                return "<!DOCTYPE html>\n"

316

                     + "<html lang='en'>\n"

317

                     + "<head>\n"

318

                     + "<title>Nanopub Query: Type Repos</title>\n"

319

                     + "<meta charset='utf-8'>\n"

320

                     + "<link rel=\"stylesheet\" href=\"/style.css\">\n"

321

                     + "</head>\n"

322

                     + "<body>\n"

323

                     + "<h3>Type Repos</h3>"

324

                     + "<p>Repos:</p>"

325

                     + "<ul>" + repos + "</ul>"

326

                     + "</body>\n"

327

                     + "</html>";

328

            }, false).onSuccess(html -> {

329

                req.response().putHeader("content-type", "text/html").end(html);

330

            }).onFailure(ex -> {

331

                req.response().setStatusCode(500).end("Error: " + ex.getMessage());

332

});

333

});

334

        io.vertx.core.Handler<RoutingContext> spacesHandler = req -> {

335

            if (!FeatureFlags.spacesEnabled()) {

336

                req.response().setStatusCode(404)

337

                        .putHeader("content-type", "text/plain")

338

                        .end("Spaces feature is disabled");

339

                return;

340

341

            // Path suffix wins over Accept header so /spaces.json is unambiguous.

342

            boolean wantJson = req.normalizedPath().endsWith(".json")

343

                    || "application/json".equalsIgnoreCase(req.request().getHeader("Accept"));

344

            vertx.<String>executeBlocking(() -> {

345

                var rows = SpacesListingRoute.fetchRows();

346

                return wantJson

347

                        ? SpacesListingRoute.renderJson(rows)

348

                        : SpacesListingRoute.renderHtml(rows);

349

            }, false).onSuccess(body -> {

350

                req.response().putHeader(

351

                        "content-type",

352

                        wantJson ? "application/json" : "text/html").end(body);

353

            }).onFailure(ex -> {

354

                req.response().setStatusCode(500).end("Error: " + ex.getMessage());

355

});

356

};

357

        proxyRouter.route(HttpMethod.GET, "/spaces").handler(spacesHandler);

358

        proxyRouter.route(HttpMethod.GET, "/spaces.json").handler(spacesHandler);

359

        proxyRouter.route(HttpMethod.GET, "/style.css").handler(req -> {

360

            if (css == null) {

361

                css = getResourceAsString("style.css");

362

363

            req.response().end(css);

364

});

365

366

        // TODO This is no longer needed and can be removed at some point:

367

        proxyRouter.route(HttpMethod.GET, "/grlc-spec/*").handler(req -> {

368

            vertx.<String>executeBlocking(() -> {

369

                GrlcSpec gsp = new GrlcSpec(req.normalizedPath(), req.queryParams());

370

                return gsp.getSpec();

371

            }, false).onSuccess(spec -> {

372

                req.response().putHeader("content-type", "text/yaml").end(spec);

373

            }).onFailure(ex -> {

374

                if (ex instanceof InvalidGrlcSpecException) {

375

                    req.response().setStatusCode(400).end(ex.getMessage());

376

                } else {

377

                    req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());

378

379

});

380

});

381

382

        proxyRouter.route(HttpMethod.GET, "/openapi/spec/*").handler(req -> {

383

            vertx.<String>executeBlocking(() -> {

384

                OpenApiSpecPage osp = new OpenApiSpecPage(req.normalizedPath(), req.queryParams());

385

                return osp.getSpec();

386

            }, false).onSuccess(spec -> {

387

                req.response().putHeader("content-type", "text/yaml").end(spec);

388

            }).onFailure(ex -> {

389

                if (ex instanceof InvalidGrlcSpecException) {

390

                    req.response().setStatusCode(400).end("Invalid grlc API definition: " + ex.getMessage());

391

                } else {

392

                    req.response().setStatusCode(500).end("Unexpected error: " + ex.getMessage());

393

394

});

395

});

396

397

        proxyRouter.route("/openapi/*").handler(StaticHandler.create("com/knowledgepixels/query/swagger"));

398

399

        HttpProxy grlcxProxy = HttpProxy.reverseProxy(httpClient);

400

        grlcxProxy.origin(proxyPort, proxy);

401

403

404

            @Override

405

            @GeneratedFlagForDependentElements

406

            public Future<ProxyResponse> handleProxyRequest(ProxyContext context) {

407

                final ProxyRequest req = context.request();

408

                final String apiPattern = "^/api/(RA[a-zA-Z0-9-_]{43})/([a-zA-Z0-9-_]+)([.]csv|[.]json|[.]srx)?([?].*)?$";

409

                if (req.getURI().matches(apiPattern)) {

410

                    try {

411

                        req.setMethod(HttpMethod.POST);

412

                        if (req.getURI().matches(".*[.]csv([?].*)?$")) {

413

                            req.putHeader("Accept", "text/csv");

414

                            req.setURI(req.getURI().replaceFirst("[.]csv([?].*)?$", "$1"));

415

                        } else if (req.getURI().matches(".*[.]json([?].*)?$")) {

416

                            req.putHeader("Accept", "application/json");

417

                            req.setURI(req.getURI().replaceFirst("[.]json([?].*)?$", "$1"));

418

                        } else if (req.getURI().matches(".*[.]srx([?].*)?$")) {

419

                            req.putHeader("Accept", "application/xml");

420

                            req.setURI(req.getURI().replaceFirst("[.]srx([?].*)?$", "$1"));

421

422

                        GrlcSpec grlcSpec = new GrlcSpec(req.getURI(), req.proxiedRequest().params());

423

424

                        // Variant 1:

425

                        req.putHeader("Content-Type", "application/sparql-query");

426

                        req.setBody(Body.body(Buffer.buffer(grlcSpec.expandQuery())));

427

                        // Variant 2:

428

                        //req.putHeader("Content-Type", "application/x-www-form-urlencoded");

429

                        //req.setBody(Body.body(Buffer.buffer("query=" + URLEncoder.encode(grlcSpec.getExpandedQueryContent(), Charsets.UTF_8))));

430

431

                        req.setURI("/rdf4j-server/repositories/" + grlcSpec.getRepoName());

432

                        logger.info("Forwarding apix request to /rdf4j-server/repositories/{}", grlcSpec.getRepoName());

433

                    } catch (InvalidGrlcSpecException ex) {

434

                        return Future.succeededFuture(context.request()

435

                                .response()

436

                                .setStatusCode(400)

437

                                .putHeader("Content-Type", "text/plain")

438

                                .setBody(Body.body(Buffer.buffer("Bad request: " + ex.getMessage()))));

439

                    } catch (Exception ex) {

440

                        return Future.succeededFuture(context.request()

441

                                .response()

442

                                .setStatusCode(500)

443

                                .putHeader("Content-Type", "text/plain")

444

                                .setBody(Body.body(Buffer.buffer("Unexpected error: " + ex.getMessage()))));

445

446

447

                return ProxyInterceptor.super.handleProxyRequest(context);

448

449

450

            @Override

451

            @GeneratedFlagForDependentElements

452

            public Future<Void> handleProxyResponse(ProxyContext context) {

453

                logger.info("Receiving api response");

454

                ProxyResponse resp = context.response();

455

                resp.putHeader("Access-Control-Allow-Origin", "*");

456

                resp.putHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS");

457

                resp.putHeader("Content-Disposition", "inline");

458

                return ProxyInterceptor.super.handleProxyResponse(context);

459

460

461

});

462

        proxyRouter.route(HttpMethod.GET, "/api/*").handler(ProxyHandler.create(grlcxProxy));

463

464

        // Handle HEAD requests for all paths not already covered (e.g. /repo/* has its own HEAD handler).

465

        // Global headers are applied before routing, so we just end the response with no body.

466

        proxyRouter.route(HttpMethod.HEAD, "/*").handler(req -> {

467

            req.response().setStatusCode(200).end();

468

});

469

470

        proxyServer.requestHandler(req -> {

471

            applyGlobalHeaders(req.response());

472

            proxyRouter.handle(req);

473

});

474

        proxyServer.listen(9393);

475

476

        // Periodic metrics update. Runs on a dedicated single-thread scheduled executor

477

        // (not on the Vert.x event loop) because `updateMetrics` can fall through to a

478

        // synchronous HTTP call in `TripleStore.getRepositoryNames()` when the cache has

479

        // been invalidated. `scheduleWithFixedDelay` naturally serialises ticks and cannot

480

        // pile up if the work occasionally runs long. Same pattern as `JellyNanopubLoader.loadUpdates`

481

        // below.

482

        Executors.newSingleThreadScheduledExecutor()

483

                .scheduleWithFixedDelay(collector::updateMetrics, 1, 1, TimeUnit.SECONDS);

484

485

486

        new Thread(() -> {

487

            try {

488

                var status = StatusController.get().initialize();

489

                logger.info("Current state: {}, last committed counter: {}", status.state, status.loadCounter);

490

                // Restore or fetch the registry setup ID

491

                Long storedSetupId = StatusController.get().getRegistrySetupId();

492

                if (storedSetupId != null) {

493

                    JellyNanopubLoader.setLastKnownSetupId(storedSetupId);

494

                    logger.info("Restored registry setupId: {}", storedSetupId);

495

                } else if (status.state == StatusController.State.LAUNCHING

496

                        || status.state == StatusController.State.LOADING_INITIAL) {

497

                    // Fresh start or crashed during initial load – safe to adopt the current setupId

498

                    try {

499

                        var metadata = JellyNanopubLoader.fetchRegistryMetadata();

500

                        JellyNanopubLoader.setLastKnownSetupId(metadata.setupId());

501

                        if (metadata.setupId() != null) {

502

                            StatusController.get().setRegistrySetupId(metadata.setupId());

503

                            logger.info("Fetched initial registry setupId: {}", metadata.setupId());

504

505

                    } catch (Exception e) {

506

                        logger.warn("Could not fetch initial registry setupId", e);

507

508

                } else {

509

                    // Upgrade from a version without setupId tracking. The DB has data but

510

                    // we can't verify it matches the current registry state. Leave lastKnownSetupId

511

                    // as null so that loadUpdates() will trigger a resync.

512

                    logger.warn("No stored registry setupId but DB has data (state: {}, counter: {}). "

513

                            + "A resync will be triggered on the first update poll.",

514

                            status.state, status.loadCounter);

515

516

                boolean forceResync = "true".equalsIgnoreCase(

517

                        Utils.getEnvString("FORCE_RESYNC", "false"));

518

                if (forceResync && status.state != StatusController.State.LAUNCHING) {

519

                    logger.warn("FORCE_RESYNC is set. Forcing full re-load from registry.");

520

                    var metadata = JellyNanopubLoader.fetchRegistryMetadata();

521

                    JellyNanopubLoader.setLastKnownSetupId(metadata.setupId());

522

                    if (metadata.setupId() != null) {

523

                        StatusController.get().setRegistrySetupId(metadata.setupId());

524

525

                    StatusController.get().setResetting();

526

                    StatusController.get().setLoadingInitial(-1);

527

                    JellyNanopubLoader.loadInitial(-1);

528

                    StatusController.get().setReady();

529

                } else if (status.state == StatusController.State.LAUNCHING || status.state == StatusController.State.LOADING_INITIAL) {

530

                    // Do the initial nanopublication loading

531

                    StatusController.get().setLoadingInitial(status.loadCounter);

532

                    // Fall back to local nanopub loading if the local files are present

533

                    if (!LocalNanopubLoader.init()) {

534

                        JellyNanopubLoader.loadInitial(status.loadCounter);

535

                    } else {

536

                        logger.info("Local nanopublication loading finished");

537

538

                    StatusController.get().setReady();

539

                } else {

540

                    logger.info("Initial load is already done");

541

                    StatusController.get().setReady();

542

543

            } catch (Exception ex) {

544

                logger.info("Initial load failed, terminating...", ex);

545

                Runtime.getRuntime().exit(1);

546

547

548

            // Seed the TrustStateRegistry from any persisted pointer before the

549

            // periodic poll begins, so the first tick doesn't re-materialize state

550

            // we already have.

551

            TrustStateLoader.bootstrap();

552

553

            // Drop any npass:* graph that isn't the current-pointer target —

554

            // leftovers from builds interrupted by a crash.

555

            if (FeatureFlags.spacesEnabled()) {

556

                AuthorityResolver.get().cleanOrphans();

557

558

559

            // Start periodic nanopub loading

560

            logger.info("Starting periodic nanopub loading...");

561

            var executor = Executors.newSingleThreadScheduledExecutor();

562

            executor.scheduleWithFixedDelay(

563

                    JellyNanopubLoader::loadUpdates,

564

                    JellyNanopubLoader.UPDATES_POLL_INTERVAL,

565

                    JellyNanopubLoader.UPDATES_POLL_INTERVAL,

566

                    TimeUnit.MILLISECONDS

567

);

568

569

            // Periodic shard-consistency sweep (issue #139): verifies that recently

570

            // loaded nanopubs actually landed in every shard repo their metadata

571

            // implies, and re-loads any that are missing. Own single-threaded

572

            // executor; scheduleWithFixedDelay serialises ticks. The tick itself

573

            // no-ops unless the reconciliation flag is on and the state is READY.

574

            Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(

575

                    () -> {

576

                        try {

577

                            ShardReconciler.tick();

578

                        } catch (Exception ex) {

579

                            logger.warn("Shard reconciliation tick failed", ex);

580

581

},

582

                    15, 15, TimeUnit.MINUTES

583

);

584

585

            // Periodic authority-resolver tick: detects trust-state flips and

586

            // advances the current space-state graph by an incremental cycle on

587

            // each load-number delta. Same cadence as the nanopub-loading poll.

588

//

589

            // The same single-threaded executor also runs periodicRebuildTick

590

            // every 10 min; that's the from-scratch rebuild triggered when an

591

            // incremental cycle DELETEs a structural derivation and raises the

592

            // npa:needsFullRebuild flag. Sharing one executor serialises the

593

            // two ticks naturally — they never overlap.

594

            if (FeatureFlags.spacesEnabled()) {

595

                var spacesExecutor = Executors.newSingleThreadScheduledExecutor();

596

                spacesExecutor.scheduleWithFixedDelay(

597

                        () -> {

598

                            try {

599

                                AuthorityResolver.get().tick();

600

                            } catch (Exception ex) {

601

                                logger.warn("AuthorityResolver tick failed", ex);

602

603

},

604

                        JellyNanopubLoader.UPDATES_POLL_INTERVAL,

605

                        JellyNanopubLoader.UPDATES_POLL_INTERVAL,

606

                        TimeUnit.MILLISECONDS

607

);

608

                spacesExecutor.scheduleWithFixedDelay(

609

                        () -> {

610

                            try {

611

                                AuthorityResolver.get().periodicRebuildTick();

612

                            } catch (Exception ex) {

613

                                logger.warn("AuthorityResolver periodic rebuild failed", ex);

614

615

},

616

                        10, 10, TimeUnit.MINUTES

617

);

618

619

        }).start();

620

621

        Runtime.getRuntime().addShutdownHook(new Thread(() -> {

622

            try {

623

                logger.info("Gracefully shutting down...");

624

                TripleStore.get().shutdownRepositories();

625

                vertx.close().toCompletionStage().toCompletableFuture().get(5, TimeUnit.SECONDS);

626

                logger.info("Graceful shutdown completed");

627

            } catch (Exception ex) {

628

                logger.info("Graceful shutdown failed", ex);

629

630

        }));

631

632

633

    private String getResourceAsString(String file) {

634

        InputStream is = getClass().getClassLoader().getResourceAsStream("com/knowledgepixels/query/" + file);

635

        try (Scanner s = new Scanner(is).useDelimiter("\\A")) {

636

            String fileContent = s.hasNext() ? s.next() : "";

637

            return fileContent;

638

639

640

641

    private static void handleRedirect(RoutingContext req, String path) {

642

        String queryString = "";

643

        if (!req.queryParam("query").isEmpty())

644

            queryString = "?query=" + URLEncoder.encode(req.queryParam("query").getFirst(), Charsets.UTF_8);

645

        if (req.queryParam("for-type").size() == 1) {

646

            String type = req.queryParam("for-type").getFirst();

647

            req.response().putHeader("location", path + "/type/" + Utils.createHash(type) + queryString);

648

            req.response().setStatusCode(301).end();

649

        } else if (req.queryParam("for-pubkey").size() == 1) {

650

            String type = req.queryParam("for-pubkey").getFirst();

651

            req.response().putHeader("location", path + "/pubkey/" + Utils.createHash(type) + queryString);

652

            req.response().setStatusCode(301).end();

653

        } else if (req.queryParam("for-user").size() == 1) {

654

            String type = req.queryParam("for-user").getFirst();

655

            req.response().putHeader("location", path + "/user/" + Utils.createHash(type) + queryString);

656

            req.response().setStatusCode(301).end();

657

658

659

660

/**

661

     * Apply headers to the response that should be present for all requests.

662

663

     * @param response The response to which the headers should be applied.

664

*/

665

    static void applyGlobalHeaders(HttpServerResponse response) {

666

        var state = StatusController.get().getState();

667

        response.putHeader("Nanopub-Query-Version", Utils.getVersion());

668

        response.putHeader("Nanopub-Query-Status", state.state.toString());

669

        response.putHeader("Nanopub-Query-Registry-Url", JellyNanopubLoader.registryUrl);

670

        Long setupId = StatusController.get().getRegistrySetupId();

671

        response.putHeader("Nanopub-Query-Registry-Setup-Id", setupId == null ? "" : setupId.toString());

672

        response.putHeader("Nanopub-Query-Load-Counter", String.valueOf(state.loadCounter));

673

        // Forward registry metadata headers

674

        String coverageTypes = JellyNanopubLoader.lastCoverageTypes;

675

        response.putHeader("Nanopub-Query-Registry-Coverage-Types", coverageTypes != null ? coverageTypes : "all");

676

        String coverageAgents = JellyNanopubLoader.lastCoverageAgents;

677

        response.putHeader("Nanopub-Query-Registry-Coverage-Agents", coverageAgents != null ? coverageAgents : "viaSetting");

678

        String testInstance = JellyNanopubLoader.lastTestInstance;

679

        if (testInstance != null) {

680

            response.putHeader("Nanopub-Query-Registry-Test-Instance", testInstance);

681

682

        String nanopubCount = JellyNanopubLoader.lastNanopubCount;

683

        if (nanopubCount != null) {

684

            response.putHeader("Nanopub-Query-Registry-Nanopub-Count", nanopubCount);

685

686

        Long loadedCount = NanopubLoader.getLoadedNanopubCount();

687

        if (loadedCount != null) {

688

            response.putHeader("Nanopub-Query-Loaded-Nanopub-Count", loadedCount.toString());

689

690

        String loadedChecksum = NanopubLoader.getLoadedNanopubChecksum();

691

        if (loadedChecksum != null) {

692

            response.putHeader("Nanopub-Query-Loaded-Nanopub-Checksum", loadedChecksum);

693

694

695

knowledgepixels / nanopub-query / 30254746745

Source File
Press 'n' to go to next uncovered line, 'b' for previous

knowledgepixels / nanopub-query / 30254746745

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous