#121

Committed 26 Sep 2023 10:10PM UTC coverage: 0.0%. Remained the same

Build # #121

Build Type

push

Committed by

Commit Message

-Add CSV plugin. WIP.

Run Details

100 of 100 new or added lines in 1 file covered. (100.0%)

0 of 6376 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/main/java/com/windhoverlabs/yamcs/archive/CSVExporter.java

package com.windhoverlabs.yamcs.archive;

import com.google.protobuf.ByteString;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.yamcs.AbstractYamcsService;
import org.yamcs.InitException;
import org.yamcs.YConfiguration;
import org.yamcs.YamcsServer;
import org.yamcs.api.HttpBody;
import org.yamcs.api.Observer;
import org.yamcs.archive.ReplayOptions;
import org.yamcs.client.YamcsClient;
import org.yamcs.client.archive.ArchiveClient;
import org.yamcs.http.BadRequestException;
import org.yamcs.http.MediaType;
import org.yamcs.http.api.ManagementApi;
import org.yamcs.http.api.ParameterReplayListener;
import org.yamcs.http.api.ReplayFactory;
// import org.yamcs.http.api.StreamArchiveApi.CsvParameterStreamer;
import org.yamcs.mdb.XtceDbFactory;
import org.yamcs.parameter.ParameterValueWithId;
import org.yamcs.protobuf.Archive.ExportParameterValuesRequest;
import org.yamcs.protobuf.Yamcs.NamedObjectId;
import org.yamcs.protobuf.Yamcs.ParameterReplayRequest;
import org.yamcs.security.ObjectPrivilegeType;
import org.yamcs.security.User;
import org.yamcs.utils.ParameterFormatter;
import org.yamcs.utils.TimeEncoding;
import org.yamcs.xtce.Parameter;
import org.yamcs.xtce.XtceDb;

public class CSVExporter extends AbstractYamcsService {

  private static class CsvParameterStreamer extends ParameterReplayListener {

    Observer<HttpBody> observer;
    List<NamedObjectId> ids;
    boolean addRaw;
    boolean addMonitoring;
    int recordCount = 0;
    char columnDelimiter = '\t';

    CsvParameterStreamer(
        Observer<HttpBody> observer,
        String filename,
        List<NamedObjectId> ids,
        boolean addRaw,
        boolean addMonitoring) {
      this.observer = observer;
      this.ids = ids;
      this.addRaw = addRaw;
      this.addMonitoring = addMonitoring;

      HttpBody metadata =
          HttpBody.newBuilder()
              .setContentType(MediaType.CSV.toString())
              .setFilename(filename)
              .build();

      observer.next(metadata);
    }

    @Override
    protected void onParameterData(List<ParameterValueWithId> params) {

      ByteString.Output data = ByteString.newOutput();
      try (Writer writer = new OutputStreamWriter(data, StandardCharsets.UTF_8);
          ParameterFormatter formatter = new ParameterFormatter(writer, ids, columnDelimiter)) {
        formatter.setWriteHeader(recordCount == 0);
        formatter.setPrintRaw(addRaw);
        formatter.setPrintMonitoring(addMonitoring);
        formatter.writeParameters(params);
      } catch (IOException e) {
        throw new UncheckedIOException(e);
      }

      HttpBody body = HttpBody.newBuilder().setData(data.toByteString()).build();
      observer.next(body);
      recordCount++;
    }

    @Override
    public void replayFailed(Throwable t) {
      observer.completeExceptionally(t);
    }

    @Override
    public void replayFinished() {
      observer.complete();
    }
  }

  private YamcsClient yamcsClient;
  private ArchiveClient archiveClient;
  private boolean realtime;
  private String start;
  private String stop;

  @Override
  public void init(String yamcsInstance, String serviceName, YConfiguration config)
      throws InitException {
    yamcsClient = YamcsClient.newBuilder("http://localhost", 8090).build();
    archiveClient = yamcsClient.createArchiveClient(yamcsInstance);

    realtime =
        this.config.getBoolean(
            "realtime", false); // might be useful for "always" writing to a CSV file...
    start = this.config.getString("start");
    stop = this.config.getString("stop");
  }

  @Override
  protected void doStart() {
    //                archiveClient
    // TODO Auto-generated method stub
    this.export(
        YamcsServer.getServer().getSecurityStore().getDirectory().getUser("admin"),
        ExportParameterValuesRequest.newBuilder().build(),
        null);
  }

  @Override
  protected void doStop() {
    // TODO Auto-generated method stub

  }

  private void export(
      User user, ExportParameterValuesRequest request, Observer<HttpBody> observer) {
    String instance = ManagementApi.verifyInstance(request.getInstance());
    //
    ReplayOptions repl = ReplayOptions.getAfapReplay();
    //
    List<NamedObjectId> ids = new ArrayList<>();
    XtceDb mdb = XtceDbFactory.getInstance(instance);
    String namespace = null;

    if (request.hasStart()) {
      repl.setRangeStart(TimeEncoding.fromProtobufTimestamp(request.getStart()));
    }
    if (request.hasStop()) {
      repl.setRangeStop(TimeEncoding.fromProtobufTimestamp(request.getStop()));
    }
    //    TODO:Get names from conofig/api request
    for (String id : request.getParametersList()) {
      //        ParameterWithId paramWithId = MdbApi.verifyParameterWithId(ctx, mdb, id);
      NamedObjectId.newBuilder().build();
      ids.add(NamedObjectId.newBuilder().build());
    }
    if (request.hasNamespace()) {
      namespace = request.getNamespace();
    }

    if (ids.isEmpty()) {
      for (Parameter p : mdb.getParameters()) {
        if (!user.hasObjectPrivilege(ObjectPrivilegeType.ReadParameter, p.getQualifiedName())) {
          continue;
        }
        if (namespace != null) {
          String alias = p.getAlias(namespace);
          if (alias != null) {
            ids.add(NamedObjectId.newBuilder().setNamespace(namespace).setName(alias).build());
          }
        } else {
          ids.add(NamedObjectId.newBuilder().setName(p.getQualifiedName()).build());
        }
      }
    }
    repl.setParameterRequest(ParameterReplayRequest.newBuilder().addAllNameFilter(ids).build());

    String filename;
    String dateString = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date());
    if (ids.size() == 1) {
      NamedObjectId id = ids.get(0);
      String parameterName = id.hasNamespace() ? id.getName() : id.getName().substring(1);
      filename = parameterName.replace('/', '_') + "_export_" + dateString + ".csv";
    } else {
      filename = "parameter_export_" + dateString + ".csv";
    }

    boolean addRaw = false;
    boolean addMonitoring = false;
    for (String extra : request.getExtraList()) {
      if (extra.equals("raw")) {
        addRaw = true;
      } else if (extra.equals("monitoring")) {
        addMonitoring = true;
      } else {
        throw new BadRequestException("Unexpected option for parameter 'extra': " + extra);
      }
    }
    CsvParameterStreamer l =
        new CsvParameterStreamer(observer, filename, ids, addRaw, addMonitoring);
    if (request.hasDelimiter()) {
      switch (request.getDelimiter()) {
        case "TAB":
          l.columnDelimiter = '\t';
          break;
        case "SEMICOLON":
          l.columnDelimiter = ';';
          break;
        case "COMMA":
          l.columnDelimiter = ',';
          break;
        default:
          throw new BadRequestException("Unexpected column delimiter");
      }
    }
    observer.setCancelHandler(l::requestReplayAbortion);
    ReplayFactory.replay(instance, user, repl, l);
  }
}

1	package com.windhoverlabs.yamcs.archive;
2
3	import com.google.protobuf.ByteString;
4	import java.io.IOException;
5	import java.io.OutputStreamWriter;
6	import java.io.UncheckedIOException;
7	import java.io.Writer;
8	import java.nio.charset.StandardCharsets;
9	import java.text.SimpleDateFormat;
10	import java.util.ArrayList;
11	import java.util.Date;
12	import java.util.List;
13	import org.yamcs.AbstractYamcsService;
14	import org.yamcs.InitException;
15	import org.yamcs.YConfiguration;
16	import org.yamcs.YamcsServer;
17	import org.yamcs.api.HttpBody;
18	import org.yamcs.api.Observer;
19	import org.yamcs.archive.ReplayOptions;
20	import org.yamcs.client.YamcsClient;
21	import org.yamcs.client.archive.ArchiveClient;
22	import org.yamcs.http.BadRequestException;
23	import org.yamcs.http.MediaType;
24	import org.yamcs.http.api.ManagementApi;
25	import org.yamcs.http.api.ParameterReplayListener;
26	import org.yamcs.http.api.ReplayFactory;
27	// import org.yamcs.http.api.StreamArchiveApi.CsvParameterStreamer;
28	import org.yamcs.mdb.XtceDbFactory;
29	import org.yamcs.parameter.ParameterValueWithId;
30	import org.yamcs.protobuf.Archive.ExportParameterValuesRequest;
31	import org.yamcs.protobuf.Yamcs.NamedObjectId;
32	import org.yamcs.protobuf.Yamcs.ParameterReplayRequest;
33	import org.yamcs.security.ObjectPrivilegeType;
34	import org.yamcs.security.User;
35	import org.yamcs.utils.ParameterFormatter;
36	import org.yamcs.utils.TimeEncoding;
37	import org.yamcs.xtce.Parameter;
38	import org.yamcs.xtce.XtceDb;
39
40	public class CSVExporter extends AbstractYamcsService {	×
41
42	private static class CsvParameterStreamer extends ParameterReplayListener {
43
44	Observer<HttpBody> observer;
45	List<NamedObjectId> ids;
46	boolean addRaw;
47	boolean addMonitoring;
48	int recordCount = 0;	×
49	char columnDelimiter = '\t';	×
50
51	CsvParameterStreamer(
52	Observer<HttpBody> observer,
53	String filename,
54	List<NamedObjectId> ids,
55	boolean addRaw,
56	boolean addMonitoring) {	×
57	this.observer = observer;	×
58	this.ids = ids;	×
59	this.addRaw = addRaw;	×
60	this.addMonitoring = addMonitoring;	×
61
62	HttpBody metadata =
63	HttpBody.newBuilder()	×
64	.setContentType(MediaType.CSV.toString())	×
65	.setFilename(filename)	×
66	.build();	×
67
68	observer.next(metadata);	×
69	}	×
70
71	@Override
72	protected void onParameterData(List<ParameterValueWithId> params) {
73
74	ByteString.Output data = ByteString.newOutput();	×
75	try (Writer writer = new OutputStreamWriter(data, StandardCharsets.UTF_8);	×
76	ParameterFormatter formatter = new ParameterFormatter(writer, ids, columnDelimiter)) {	×
77	formatter.setWriteHeader(recordCount == 0);	×
78	formatter.setPrintRaw(addRaw);	×
79	formatter.setPrintMonitoring(addMonitoring);	×
80	formatter.writeParameters(params);	×
81	} catch (IOException e) {	×
82	throw new UncheckedIOException(e);	×
83	}	×
84
85	HttpBody body = HttpBody.newBuilder().setData(data.toByteString()).build();	×
86	observer.next(body);	×
87	recordCount++;	×
88	}	×
89
90	@Override
91	public void replayFailed(Throwable t) {
92	observer.completeExceptionally(t);	×
93	}	×
94
95	@Override
96	public void replayFinished() {
97	observer.complete();	×
98	}	×
99	}
100
101	private YamcsClient yamcsClient;
102	private ArchiveClient archiveClient;
103	private boolean realtime;
104	private String start;
105	private String stop;
106
107	@Override
108	public void init(String yamcsInstance, String serviceName, YConfiguration config)
109	throws InitException {
110	yamcsClient = YamcsClient.newBuilder("http://localhost", 8090).build();	×
111	archiveClient = yamcsClient.createArchiveClient(yamcsInstance);	×
112
113	realtime =	×
114	this.config.getBoolean(	×
115	"realtime", false); // might be useful for "always" writing to a CSV file...
116	start = this.config.getString("start");	×
117	stop = this.config.getString("stop");	×
118	}	×
119
120	@Override
121	protected void doStart() {
122	// archiveClient
123	// TODO Auto-generated method stub
124	this.export(	×
125	YamcsServer.getServer().getSecurityStore().getDirectory().getUser("admin"),	×
126	ExportParameterValuesRequest.newBuilder().build(),	×
127	null);
128	}	×
129
130	@Override
131	protected void doStop() {
132	// TODO Auto-generated method stub
133
134	}	×
135
136	private void export(
137	User user, ExportParameterValuesRequest request, Observer<HttpBody> observer) {
138	String instance = ManagementApi.verifyInstance(request.getInstance());	×
139	//
140	ReplayOptions repl = ReplayOptions.getAfapReplay();	×
141	//
142	List<NamedObjectId> ids = new ArrayList<>();	×
143	XtceDb mdb = XtceDbFactory.getInstance(instance);	×
144	String namespace = null;	×
145
146	if (request.hasStart()) {	×
147	repl.setRangeStart(TimeEncoding.fromProtobufTimestamp(request.getStart()));	×
148	}
149	if (request.hasStop()) {	×
150	repl.setRangeStop(TimeEncoding.fromProtobufTimestamp(request.getStop()));	×
151	}
152	// TODO:Get names from conofig/api request
153	for (String id : request.getParametersList()) {	×
154	// ParameterWithId paramWithId = MdbApi.verifyParameterWithId(ctx, mdb, id);
155	NamedObjectId.newBuilder().build();	×
156	ids.add(NamedObjectId.newBuilder().build());	×
157	}	×
158	if (request.hasNamespace()) {	×
159	namespace = request.getNamespace();	×
160	}
161
162	if (ids.isEmpty()) {	×
163	for (Parameter p : mdb.getParameters()) {	×
164	if (!user.hasObjectPrivilege(ObjectPrivilegeType.ReadParameter, p.getQualifiedName())) {	×
165	continue;	×
166	}
167	if (namespace != null) {	×
168	String alias = p.getAlias(namespace);	×
169	if (alias != null) {	×
170	ids.add(NamedObjectId.newBuilder().setNamespace(namespace).setName(alias).build());	×
171	}
172	} else {	×
173	ids.add(NamedObjectId.newBuilder().setName(p.getQualifiedName()).build());	×
174	}
175	}	×
176	}
177	repl.setParameterRequest(ParameterReplayRequest.newBuilder().addAllNameFilter(ids).build());	×
178
179	String filename;
180	String dateString = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date());	×
181	if (ids.size() == 1) {	×
182	NamedObjectId id = ids.get(0);	×
183	String parameterName = id.hasNamespace() ? id.getName() : id.getName().substring(1);	×
184	filename = parameterName.replace('/', '_') + "_export_" + dateString + ".csv";	×
185	} else {	×
186	filename = "parameter_export_" + dateString + ".csv";	×
187	}
188
189	boolean addRaw = false;	×
190	boolean addMonitoring = false;	×
191	for (String extra : request.getExtraList()) {	×
192	if (extra.equals("raw")) {	×
193	addRaw = true;	×
194	} else if (extra.equals("monitoring")) {	×
195	addMonitoring = true;	×
196	} else {
197	throw new BadRequestException("Unexpected option for parameter 'extra': " + extra);	×
198	}
199	}	×
200	CsvParameterStreamer l =	×
201	new CsvParameterStreamer(observer, filename, ids, addRaw, addMonitoring);
202	if (request.hasDelimiter()) {	×
203	switch (request.getDelimiter()) {	×
204	case "TAB":
205	l.columnDelimiter = '\t';	×
206	break;	×
207	case "SEMICOLON":
208	l.columnDelimiter = ';';	×
209	break;	×
210	case "COMMA":
211	l.columnDelimiter = ',';	×
212	break;	×
213	default:
214	throw new BadRequestException("Unexpected column delimiter");	×
215	}
216	}
217	observer.setCancelHandler(l::requestReplayAbortion);	×
218	ReplayFactory.replay(instance, user, repl, l);	×
219	}	×
220	}

WindhoverLabs / yamcs-cfs / #121

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous