• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

WindhoverLabs / yamcs-cfs / #126

27 Sep 2023 05:01PM UTC coverage: 0.0%. Remained the same
#126

push

lorenzo-gomez-windhover
-Update docs

0 of 6576 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0
/src/main/java/com/windhoverlabs/yamcs/archive/CSVExporter.java
1
package com.windhoverlabs.yamcs.archive;
2

3
import com.google.protobuf.ByteString;
4
import com.google.protobuf.Timestamp;
5
import java.io.File;
6
import java.io.FileNotFoundException;
7
import java.io.FileOutputStream;
8
import java.io.IOException;
9
import java.io.OutputStreamWriter;
10
import java.io.UncheckedIOException;
11
import java.io.Writer;
12
import java.nio.charset.StandardCharsets;
13
import java.nio.file.Path;
14
import java.nio.file.Paths;
15
import java.text.SimpleDateFormat;
16
import java.time.Instant;
17
import java.util.ArrayList;
18
import java.util.Date;
19
import java.util.HashMap;
20
import java.util.List;
21
import java.util.Map;
22
import java.util.Map.Entry;
23
import org.yamcs.AbstractYamcsService;
24
import org.yamcs.InitException;
25
import org.yamcs.YConfiguration;
26
import org.yamcs.YamcsServer;
27
import org.yamcs.api.HttpBody;
28
import org.yamcs.api.Observer;
29
import org.yamcs.archive.ReplayOptions;
30
import org.yamcs.client.Helpers;
31
import org.yamcs.client.YamcsClient;
32
import org.yamcs.client.archive.ArchiveClient;
33
import org.yamcs.http.BadRequestException;
34
import org.yamcs.http.MediaType;
35
import org.yamcs.http.api.ManagementApi;
36
import org.yamcs.http.api.ParameterReplayListener;
37
import org.yamcs.http.api.ReplayFactory;
38
import org.yamcs.mdb.XtceDbFactory;
39
import org.yamcs.parameter.ParameterValueWithId;
40
import org.yamcs.protobuf.Archive.ExportParameterValuesRequest;
41
import org.yamcs.protobuf.Yamcs.NamedObjectId;
42
import org.yamcs.protobuf.Yamcs.ParameterReplayRequest;
43
import org.yamcs.security.ObjectPrivilegeType;
44
import org.yamcs.security.User;
45
import org.yamcs.utils.TimeEncoding;
46
import org.yamcs.xtce.Parameter;
47
import org.yamcs.xtce.XtceDb;
48
import org.yamcs.yarch.FileSystemBucket;
49
import org.yamcs.yarch.YarchDatabase;
50
import org.yamcs.yarch.YarchDatabaseInstance;
51

52
public class CSVExporter extends AbstractYamcsService implements Runnable {
×
53

54
  private static class CsvParameterStreamer extends ParameterReplayListener {
55

56
    Observer<HttpBody> observer;
57
    List<NamedObjectId> ids;
58
    boolean addRaw;
59
    boolean addMonitoring;
60
    int recordCount = 0;
×
61
    char columnDelimiter = '\t';
×
62
    Instant firstInstant;
63

64
    CsvParameterStreamer(
65
        Observer<HttpBody> observer,
66
        String filename,
67
        List<NamedObjectId> ids,
68
        boolean addRaw,
69
        boolean addMonitoring) {
×
70
      this.observer = observer;
×
71
      this.ids = ids;
×
72
      this.addRaw = addRaw;
×
73
      this.addMonitoring = addMonitoring;
×
74

75
      HttpBody metadata =
76
          HttpBody.newBuilder()
×
77
              .setContentType(MediaType.CSV.toString())
×
78
              .setFilename(filename)
×
79
              .build();
×
80

81
      observer.next(metadata);
×
82
    }
×
83

84
    @Override
85
    protected void onParameterData(List<ParameterValueWithId> params) {
86

87
      ByteString.Output data = ByteString.newOutput();
×
88
      try (Writer writer = new OutputStreamWriter(data, StandardCharsets.UTF_8);
×
89
          DeltaCountedParameterFormatter formatter =
×
90
              new DeltaCountedParameterFormatter(writer, ids, columnDelimiter)) {
91
        if (recordCount == 0) {
×
92
          firstInstant =
×
93
              Helpers.toInstant(params.get(0).getParameterValue().toGpb().getGenerationTime());
×
94
        }
95
        formatter.setWriteHeader(recordCount == 0);
×
96
        formatter.setPrintRaw(addRaw);
×
97
        formatter.setPrintMonitoring(addMonitoring);
×
98
        formatter.setFirstInstant(firstInstant);
×
99
        formatter.writeParameters(params);
×
100
      } catch (IOException e) {
×
101
        throw new UncheckedIOException(e);
×
102
      }
×
103

104
      HttpBody body = HttpBody.newBuilder().setData(data.toByteString()).build();
×
105
      observer.next(body);
×
106
      recordCount++;
×
107
    }
×
108

109
    @Override
110
    public void replayFailed(Throwable t) {
111
      observer.completeExceptionally(t);
×
112
    }
×
113

114
    @Override
115
    public void replayFinished() {
116
      observer.complete();
×
117
    }
×
118
  }
119

120
  private YamcsClient yamcsClient;
121
  private ArchiveClient archiveClient;
122
  private boolean realtime;
123
  private Instant start;
124
  private Instant stop;
125
  private String bucketName;
126
  private FileSystemBucket bucket;
127
  private Thread thread;
128
  private HashMap<String, FileOutputStream> fisMap = new HashMap<String, FileOutputStream>();
×
129
  private Path bucketPath;
130
  private static final String CSV_NAME_POST_FIX = ".csv";
131
  private Map<String, List<String>> paramsMap = null;
×
132

133
  @Override
134
  public void init(String yamcsInstance, String serviceName, YConfiguration config)
135
      throws InitException {
136

137
    //    realtime =
138
    //        this.config.getBoolean(
139
    //            "realtime", false); // might be useful for "always" writing to a CSV file...
140
    //    start = Instant.parse("2023-09-23T23:00:00.000Z");
141
    start = Instant.parse(config.getString("start"));
×
142
    //    stop = Instant.parse("2023-09-24T00:10:00.000Z");
143
    stop = Instant.parse(config.getString("stop"));
×
144

145
    /* Read in our configuration parameters. */
146
    bucketName = config.getString("bucket");
×
147
    paramsMap = config.getMap("params");
×
148

149
    //    System.out.println("params:" + paramsMap);
150

151
    /* Iterate through the bucket names passed to us by the configuration file.  We're going to add the buckets
152
     * to our internal list so we can process them later. */
153
    YarchDatabaseInstance yarch = YarchDatabase.getInstance(YamcsServer.GLOBAL_INSTANCE);
×
154

155
    try {
156
      bucket = (FileSystemBucket) yarch.getBucket(bucketName);
×
157
    } catch (IOException e) {
×
158
      e.printStackTrace();
×
159
    }
×
160
  }
×
161

162
  @Override
163
  protected void doStart() {
164

165
    thread = new Thread(this);
×
166
    thread.start();
×
167
    notifyStarted();
×
168
  }
×
169

170
  @Override
171
  protected void doStop() {
172
    // TODO Auto-generated method stub
173

174
  }
×
175

176
  private void export(
177
      User user, ExportParameterValuesRequest request, Observer<HttpBody> observer) {
178
    String instance = ManagementApi.verifyInstance(request.getInstance());
×
179
    //
180
    ReplayOptions repl = ReplayOptions.getAfapReplay();
×
181
    //
182
    List<NamedObjectId> ids = new ArrayList<>();
×
183
    XtceDb mdb = XtceDbFactory.getInstance(instance);
×
184
    String namespace = null;
×
185

186
    if (request.hasStart()) {
×
187
      repl.setRangeStart(TimeEncoding.fromProtobufTimestamp(request.getStart()));
×
188
    }
189
    if (request.hasStop()) {
×
190
      repl.setRangeStop(TimeEncoding.fromProtobufTimestamp(request.getStop()));
×
191
    }
192
    //    TODO:Get names from conofig/api request
193
    for (String id : request.getParametersList()) {
×
194
      ids.add(NamedObjectId.newBuilder().setName(id).build());
×
195
    }
×
196
    if (request.hasNamespace()) {
×
197
      namespace = request.getNamespace();
×
198
    }
199

200
    if (ids.isEmpty()) {
×
201
      for (Parameter p : mdb.getParameters()) {
×
202
        if (!user.hasObjectPrivilege(ObjectPrivilegeType.ReadParameter, p.getQualifiedName())) {
×
203
          continue;
×
204
        }
205
        if (namespace != null) {
×
206
          String alias = p.getAlias(namespace);
×
207
          if (alias != null) {
×
208
            ids.add(NamedObjectId.newBuilder().setNamespace(namespace).setName(alias).build());
×
209
          }
210
        } else {
×
211
          ids.add(NamedObjectId.newBuilder().setName(p.getQualifiedName()).build());
×
212
        }
213
      }
×
214
    }
215
    repl.setParameterRequest(ParameterReplayRequest.newBuilder().addAllNameFilter(ids).build());
×
216

217
    String filename;
218
    String dateString = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date());
×
219
    if (ids.size() == 1) {
×
220
      NamedObjectId id = ids.get(0);
×
221
      String parameterName = id.hasNamespace() ? id.getName() : id.getName().substring(1);
×
222
      filename = parameterName.replace('/', '_') + "_export_" + dateString + ".csv";
×
223
    } else {
×
224
      filename = "parameter_export_" + dateString + ".csv";
×
225
    }
226

227
    boolean addRaw = false;
×
228
    boolean addMonitoring = false;
×
229
    for (String extra : request.getExtraList()) {
×
230
      if (extra.equals("raw")) {
×
231
        addRaw = true;
×
232
      } else if (extra.equals("monitoring")) {
×
233
        addMonitoring = true;
×
234
      } else {
235
        throw new BadRequestException("Unexpected option for parameter 'extra': " + extra);
×
236
      }
237
    }
×
238
    CsvParameterStreamer l =
×
239
        new CsvParameterStreamer(observer, filename, ids, addRaw, addMonitoring);
240
    if (request.hasDelimiter()) {
×
241
      switch (request.getDelimiter()) {
×
242
        case "TAB":
243
          l.columnDelimiter = '\t';
×
244
          break;
×
245
        case "SEMICOLON":
246
          l.columnDelimiter = ';';
×
247
          break;
×
248
        case "COMMA":
249
          l.columnDelimiter = ',';
×
250
          break;
×
251
        default:
252
          throw new BadRequestException("Unexpected column delimiter");
×
253
      }
254
    }
255
    //    observer.setCancelHandler(l::requestReplayAbortion);
256
    ReplayFactory.replay(instance, user, repl, l);
×
257
  }
×
258

259
  @Override
260
  public void run() {
261
    exportPV();
×
262
  }
×
263

264
  private void exportPV() {
265

266
    for (Entry<String, List<String>> params : paramsMap.entrySet()) {
×
267
      ArrayList<String> parameters = new ArrayList<String>();
×
268
      //      String[] nameTokens = param.split("/");
269

270
      FileOutputStream fis = openFile(getNewFilePath(params.getKey()).toAbsolutePath().toString());
×
271
      fisMap.put(getNewFilePath(params.getKey()).toAbsolutePath().toString(), fis);
×
272
      for (String param : params.getValue()) {
×
273
        //        String p = "/cfs/ppd/apps/to/TO_HK_TLM_MID.ChannelInfo[0].MessagesSent";
274
        parameters.add(param);
×
275
      }
×
276
      this.export(
×
277
          YamcsServer.getServer().getSecurityStore().getDirectory().getUser("admin"),
×
278
          ExportParameterValuesRequest.newBuilder()
×
279
              .addAllParameters(parameters)
×
280
              .setStart(
×
281
                  Timestamp.newBuilder()
×
282
                      .setNanos(start.getNano())
×
283
                      .setSeconds(start.getEpochSecond())
×
284
                      .build())
×
285
              .setStop(
×
286
                  Timestamp.newBuilder()
×
287
                      .setNanos(stop.getNano())
×
288
                      .setSeconds(stop.getEpochSecond())
×
289
                      .build())
×
290
              .setInstance("fsw")
×
291
              .build(),
×
292
          new Observer<HttpBody>() {
×
293

294
            @Override
295
            public void next(HttpBody message) {
296
              //              // TODO Auto-generated method stub
297
              writeToCSV(
×
298
                  message.getData().toByteArray(),
×
299
                  getNewFilePath(params.getKey()).toAbsolutePath().toString());
×
300
            }
×
301

302
            @Override
303
            public void completeExceptionally(Throwable t) {
304
              // TODO Auto-generated method stub
305

306
            }
×
307

308
            @Override
309
            public void complete() {
310
              // TODO Auto-generated method stub
311
              closeFile(fis);
×
312
            }
×
313
          });
314
    }
×
315
  }
×
316

317
  private void writeToCSV(byte[] data, String path) {
318

319
    writeToFile(data, fisMap.get(path));
×
320
  }
×
321

322
  private void writeToFile(byte[] data, FileOutputStream fis) {
323
    try {
324
      fis.write(data, 0, data.length);
×
325
    } catch (IOException e) {
×
326
      // TODO Auto-generated catch block
327
      e.printStackTrace();
×
328
    }
×
329
  }
×
330

331
  private void closeFile(FileOutputStream fis) {
332
    try {
333
      fis.close();
×
334
    } catch (IOException e) {
×
335
      // TODO Auto-generated catch block
336
      e.printStackTrace();
×
337
    }
×
338
  }
×
339

340
  private FileOutputStream openFile(String path) {
341
    File inputFile = new File(path);
×
342
    FileOutputStream fis = null;
×
343
    try {
344
      fis = new FileOutputStream(inputFile);
×
345
    } catch (FileNotFoundException e) {
×
346
      // TODO Auto-generated catch block
347
      e.printStackTrace();
×
348
    }
×
349
    return fis;
×
350
  }
351

352
  private Path getNewFilePath(String name) {
353
    bucketPath = Paths.get(bucket.getBucketRoot().toString()).toAbsolutePath();
×
354
    Path filePath = bucketPath.resolve(name + "_" + CSV_NAME_POST_FIX);
×
355

356
    return filePath;
×
357
  }
358
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2025 Coveralls, Inc