16054419974

Committed 03 Jul 2025 03:22PM UTC coverage: 39.757% (-0.1%) from 39.867%

Build # 16054419974

Build Type

push

github

Committed by

web-flow

Commit Message

Log poll import (#135)

* log import id during poll status

* update version for log changes

* cleanup

Run Details

3 of 14 new or added lines in 3 files covered. (21.43%)

1 existing line in 1 file now uncovered.

784 of 1972 relevant lines covered (39.76%)

0.4 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

5.68

/src/main/java/com/bynder/sdk/service/upload/FileUploader.java

/*
 * Copyright (c) 2017 Bynder B.V. All rights reserved.
 * <p>
 * Licensed under the MIT License. See LICENSE file in the project root for full license
 * information.
 */
package com.bynder.sdk.service.upload;

import com.bynder.sdk.api.BynderApi;
import com.bynder.sdk.exception.BynderUploadException;
import com.bynder.sdk.model.upload.*;
import com.bynder.sdk.query.decoder.QueryDecoder;
import com.bynder.sdk.query.upload.*;
import com.bynder.sdk.service.amazons3.AmazonS3Service;
import com.bynder.sdk.util.RXUtils;
import io.reactivex.Completable;
import io.reactivex.Observable;
import io.reactivex.Single;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
import java.util.concurrent.TimeUnit;

/**
 * Class used to upload files to Bynder.
 */
public class FileUploader {

    private static final Logger LOG = LoggerFactory.getLogger(FileUploader.class);


    private static final int MAX_CHUNK_SIZE = 1024 * 1024 * 5;

    /**
     * Max polling iterations to wait for the file to be converted.
     */
    private static final int MAX_POLLING_ITERATIONS = 120;

    /**
     * Idle time between polling iterations.
     */
    private static final int POLLING_IDLE_TIME = 2000;

    /**
     * Instance of {@link BynderApi} which handles the HTTP communication.
     */
    private final BynderApi bynderApi;

    /**
     * Instance of {@link QueryDecoder} to decode query objects into API parameters.
     */
    private final QueryDecoder queryDecoder;

    /**
     * Creates a new instance of the class.
     *
     * @param bynderApi    Instance to handle the HTTP communication with the Bynder API.
     * @param queryDecoder Query decoder.
     */
    public FileUploader(final BynderApi bynderApi, final QueryDecoder queryDecoder) {
        this.bynderApi = bynderApi;
        this.queryDecoder = queryDecoder;
    }

    /**
     * Uploads a file with the information specified in the query parameter.
     *
     * @param uploadQuery Upload query with the information to upload the file.
     * @return {@link Observable} with the {@link SaveMediaResponse} information.
     */
    public Single<SaveMediaResponse> uploadFile(final UploadQuery uploadQuery) {
        return getClosestS3Endpoint().flatMap(awsBucket -> {
            String filename = uploadQuery.getFilename();
            AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);
            return getUploadInformation(new RequestUploadQuery(filename))
                    .flatMap(uploadRequest -> uploadChunk(
                            amazonS3Service,
                            uploadRequest,
                            uploadQuery,
                            filename
                    ).count().flatMap(chunkCount -> finaliseUpload(new FinaliseUploadQuery(
                            uploadRequest.getS3File().getUploadId(),
                            uploadRequest.getS3File().getTargetId(),
                            uploadRequest.getS3Filename(),
                            chunkCount
                    ))));
        }).flatMap(importId ->
                pollProcessing(importId).andThen(saveUploadedMedia(importId, uploadQuery))
        );
    }

    public Single<UploadAdditionalMediaResponse> uploadAdditionalFile(final UploadQuery uploadQuery) {
        return getClosestS3Endpoint().flatMap(awsBucket -> {
            String filename = uploadQuery.getFilename();
            AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);
            return getUploadInformation(new RequestUploadQuery(filename))
                    .flatMap(uploadRequest -> uploadChunk(
                            amazonS3Service,
                            uploadRequest,
                            uploadQuery,
                            filename
                    ).count().flatMap(chunkCount -> finaliseUploadAdditional(new FinaliseUploadAdditionalQuery(
                            uploadRequest.getS3File().getUploadId(),
                            uploadRequest.getS3File().getTargetId(),
                            uploadRequest.getS3Filename(),
                            chunkCount
                    ), uploadQuery.getMediaId())));
        });
    }

    /**
     * Uploads a file with the information specified in the query parameter
     * while providing information on the progress of the upload via the Observable returned.
     *
     * @param uploadQuery Upload query with the information to upload the file.
     * @return {@link Observable} with the {@link UploadProgress} information.
     */
    public Observable<UploadProgress> uploadFileWithProgress(final UploadQuery uploadQuery) {
        String filename = uploadQuery.getFilename();
        return getClosestS3Endpoint().flatMapObservable(awsBucket -> {
            AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);
            return getUploadInformation(new RequestUploadQuery(filename))
                    .flatMapObservable(uploadRequest -> uploadChunk(
                            amazonS3Service,
                            uploadRequest,
                            uploadQuery,
                            filename
                    ));
        });
    }

    private Observable<UploadProgress> uploadChunk(
            AmazonS3Service amazonS3Service,
            UploadRequest uploadRequest,
            UploadQuery uploadQuery,
            String filename
    ) throws IOException {
        long fileSize = Files.size(Paths.get(uploadQuery.getFilepath()));
        UploadProgress uploadProgress = new UploadProgress(fileSize);
        return RXUtils.mapWithIndex(
                RXUtils.readFileChunks(uploadQuery.getFilepath(), MAX_CHUNK_SIZE),
                1
        ).flatMapSingle(chunk -> amazonS3Service.uploadPartToAmazon(
                chunk,
                filename,
                (int) ((fileSize - 1) / MAX_CHUNK_SIZE + 1),
                uploadRequest.getMultipartParams()
        ).andThen(registerChunk(new RegisterChunkQuery(
                chunk.getIndex(),
                uploadRequest.getS3File().getUploadId(),
                uploadRequest.getS3File().getTargetId(),
                String.format(
                        "%s/p%s",
                        uploadRequest.getS3Filename(),
                        chunk.getIndex()
                )
        ))).toSingle(() -> uploadProgress.addProgress(chunk.getValue().length)));
    }

    /**
     * Check {@link BynderApi#getPollStatus} for more information.
     */
    private Completable pollProcessing(final String importId) {
        LOG.info("Polling status for importId: " + importId);
        return getPollStatus(new PollStatusQuery(importId.split(",")))
                .delay(POLLING_IDLE_TIME, TimeUnit.MILLISECONDS)
                .repeat()
                .take(MAX_POLLING_ITERATIONS)
                .takeUntil(pollStatus -> pollStatus.processingDone(importId) || pollStatus.processingFailed(importId))
                .lastElement()
                .toSingle()
                .flatMapCompletable(pollStatus -> {
                    if (pollStatus.processingFailed(importId)) {
                        return Completable.error(new BynderUploadException("Processing media failed."));
                    }
                    return Completable.complete();
                });
    }

    /**
     * Calls {@link FileUploader#saveMedia(SaveMediaQuery)} to save the completely uploaded file in
     * Bynder.
     *
     * @param uploadQuery Upload query with the information to upload the file.
     * @return {@link Single} with the {@link SaveMediaResponse} information.
     */
    private Single<SaveMediaResponse> saveUploadedMedia(final String importId, final UploadQuery uploadQuery) {
        SaveMediaQuery saveMediaQuery = new SaveMediaQuery(importId)
                .setAudit(uploadQuery.isAudit())
                .setMetaproperties(uploadQuery.getMetaproperties());

        if (uploadQuery.getMediaId() == null) {
            // A new asset will be created for the uploaded file.
            return saveMedia(saveMediaQuery
                    .setBrandId(uploadQuery.getBrandId())
                    .setName(uploadQuery.getAssetName())
                    .setTags(uploadQuery.getTags())
            );
        } else {
            // The uploaded file will be attached to an existing asset.
            return saveMedia(saveMediaQuery
                    .setMediaId(uploadQuery.getMediaId())
            );
        }
    }

    /**
     * Check {@link BynderApi#getClosestS3Endpoint()} for more information.
     */
    private Single<String> getClosestS3Endpoint() {
        return bynderApi.getClosestS3Endpoint().singleOrError().map(RXUtils::getResponseBody);
    }

    /**
     * Check {@link BynderApi#getUploadInformation(Map)} for more information.
     */
    private Single<UploadRequest> getUploadInformation(final RequestUploadQuery requestUploadQuery) {
        Map<String, String> params = queryDecoder.decode(requestUploadQuery);
        return bynderApi.getUploadInformation(params).singleOrError().map(RXUtils::getResponseBody);
    }

    /**
     * Check {@link BynderApi#registerChunk(Map)} for more information.
     */
    private Completable registerChunk(final RegisterChunkQuery registerChunkQuery) {
        Map<String, String> params = queryDecoder.decode(registerChunkQuery);
        return bynderApi.registerChunk(params).ignoreElements();
    }

    /**
     * Check {@link BynderApi#finaliseUpload(Map)} for more information.
     */
    private Single<String> finaliseUpload(final FinaliseUploadQuery finaliseUploadQuery) {
        Map<String, String> params = queryDecoder.decode(finaliseUploadQuery);
        return bynderApi.finaliseUpload(params).singleOrError().map(RXUtils::getResponseBody)
                .map(FinaliseResponse::getImportId);
    }

    private Single<UploadAdditionalMediaResponse> finaliseUploadAdditional(final FinaliseUploadAdditionalQuery finaliseUploadQuery, String mediaId) {
        Map<String, String> params = queryDecoder.decode(finaliseUploadQuery);
        return bynderApi.finaliseUploadAdditional(mediaId, params).singleOrError().map(RXUtils::getResponseBody);
    }

    /**
     * Check {@link BynderApi#getPollStatus(Map)} for more information.
     */
    private Single<PollStatus> getPollStatus(final PollStatusQuery pollStatusQuery) {
        Map<String, String> params = queryDecoder.decode(pollStatusQuery);
        LOG.info("Polling status for: " + params);
        return bynderApi.getPollStatus(params)
                .doOnNext(response -> {
                    if (response.raw() != null && response.raw().request() != null) {
                        LOG.info("Request URL for poll: " + response.raw().request().url());
                    }
                })
                .singleOrError()
                .map(RXUtils::getResponseBody);
    }

    /**
     * Check {@link BynderApi#saveMedia(Map)} for more information.
     */
    private Single<SaveMediaResponse> saveMedia(final SaveMediaQuery saveMediaQuery) {
        Map<String, String> params = queryDecoder.decode(saveMediaQuery);
        return bynderApi.saveMedia(params).singleOrError().map(RXUtils::getResponseBody);
    }
}

1	/*
2	* Copyright (c) 2017 Bynder B.V. All rights reserved.
3	* <p>
4	* Licensed under the MIT License. See LICENSE file in the project root for full license
5	* information.
6	*/
7	package com.bynder.sdk.service.upload;
8
9	import com.bynder.sdk.api.BynderApi;
10	import com.bynder.sdk.exception.BynderUploadException;
11	import com.bynder.sdk.model.upload.*;
12	import com.bynder.sdk.query.decoder.QueryDecoder;
13	import com.bynder.sdk.query.upload.*;
14	import com.bynder.sdk.service.amazons3.AmazonS3Service;
15	import com.bynder.sdk.util.RXUtils;
16	import io.reactivex.Completable;
17	import io.reactivex.Observable;
18	import io.reactivex.Single;
19	import org.slf4j.Logger;
20	import org.slf4j.LoggerFactory;
21
22	import java.io.IOException;
23	import java.nio.file.Files;
24	import java.nio.file.Paths;
25	import java.util.Map;
26	import java.util.concurrent.TimeUnit;
27
28	/**
29	* Class used to upload files to Bynder.
30	*/
31	public class FileUploader {
32
33	private static final Logger LOG = LoggerFactory.getLogger(FileUploader.class);	1✔
34
35
36	private static final int MAX_CHUNK_SIZE = 1024 * 1024 * 5;
37
38	/**
39	* Max polling iterations to wait for the file to be converted.
40	*/
41	private static final int MAX_POLLING_ITERATIONS = 120;
42
43	/**
44	* Idle time between polling iterations.
45	*/
46	private static final int POLLING_IDLE_TIME = 2000;
47
48	/**
49	* Instance of {@link BynderApi} which handles the HTTP communication.
50	*/
51	private final BynderApi bynderApi;
52
53	/**
54	* Instance of {@link QueryDecoder} to decode query objects into API parameters.
55	*/
56	private final QueryDecoder queryDecoder;
57
58	/**
59	* Creates a new instance of the class.
60	*
61	* @param bynderApi Instance to handle the HTTP communication with the Bynder API.
62	* @param queryDecoder Query decoder.
63	*/
64	public FileUploader(final BynderApi bynderApi, final QueryDecoder queryDecoder) {	1✔
65	this.bynderApi = bynderApi;	1✔
66	this.queryDecoder = queryDecoder;	1✔
67	}	1✔
68
69	/**
70	* Uploads a file with the information specified in the query parameter.
71	*
72	* @param uploadQuery Upload query with the information to upload the file.
73	* @return {@link Observable} with the {@link SaveMediaResponse} information.
74	*/
75	public Single<SaveMediaResponse> uploadFile(final UploadQuery uploadQuery) {
76	return getClosestS3Endpoint().flatMap(awsBucket -> {	×
77	String filename = uploadQuery.getFilename();	×
78	AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);	×
79	return getUploadInformation(new RequestUploadQuery(filename))	×
80	.flatMap(uploadRequest -> uploadChunk(	×
81	amazonS3Service,
82	uploadRequest,
83	uploadQuery,
84	filename
85	).count().flatMap(chunkCount -> finaliseUpload(new FinaliseUploadQuery(	×
86	uploadRequest.getS3File().getUploadId(),	×
87	uploadRequest.getS3File().getTargetId(),	×
88	uploadRequest.getS3Filename(),	×
89	chunkCount
90	))));
91	}).flatMap(importId ->	×
92	pollProcessing(importId).andThen(saveUploadedMedia(importId, uploadQuery))	×
93	);
94	}
95
96	public Single<UploadAdditionalMediaResponse> uploadAdditionalFile(final UploadQuery uploadQuery) {
97	return getClosestS3Endpoint().flatMap(awsBucket -> {	×
98	String filename = uploadQuery.getFilename();	×
99	AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);	×
100	return getUploadInformation(new RequestUploadQuery(filename))	×
101	.flatMap(uploadRequest -> uploadChunk(	×
102	amazonS3Service,
103	uploadRequest,
104	uploadQuery,
105	filename
106	).count().flatMap(chunkCount -> finaliseUploadAdditional(new FinaliseUploadAdditionalQuery(	×
107	uploadRequest.getS3File().getUploadId(),	×
108	uploadRequest.getS3File().getTargetId(),	×
109	uploadRequest.getS3Filename(),	×
110	chunkCount
111	), uploadQuery.getMediaId())));	×
112	});
113	}
114
115	/**
116	* Uploads a file with the information specified in the query parameter
117	* while providing information on the progress of the upload via the Observable returned.
118	*
119	* @param uploadQuery Upload query with the information to upload the file.
120	* @return {@link Observable} with the {@link UploadProgress} information.
121	*/
122	public Observable<UploadProgress> uploadFileWithProgress(final UploadQuery uploadQuery) {
123	String filename = uploadQuery.getFilename();	×
124	return getClosestS3Endpoint().flatMapObservable(awsBucket -> {	×
125	AmazonS3Service amazonS3Service = AmazonS3Service.Builder.create(awsBucket);	×
126	return getUploadInformation(new RequestUploadQuery(filename))	×
127	.flatMapObservable(uploadRequest -> uploadChunk(	×
128	amazonS3Service,
129	uploadRequest,
130	uploadQuery,
131	filename
132	));
133	});
134	}
135
136	private Observable<UploadProgress> uploadChunk(
137	AmazonS3Service amazonS3Service,
138	UploadRequest uploadRequest,
139	UploadQuery uploadQuery,
140	String filename
141	) throws IOException {
142	long fileSize = Files.size(Paths.get(uploadQuery.getFilepath()));	×
143	UploadProgress uploadProgress = new UploadProgress(fileSize);	×
144	return RXUtils.mapWithIndex(	×
145	RXUtils.readFileChunks(uploadQuery.getFilepath(), MAX_CHUNK_SIZE),	×
146	1
147	).flatMapSingle(chunk -> amazonS3Service.uploadPartToAmazon(	×
148	chunk,
149	filename,
150	(int) ((fileSize - 1) / MAX_CHUNK_SIZE + 1),
151	uploadRequest.getMultipartParams()	×
152	).andThen(registerChunk(new RegisterChunkQuery(	×
153	chunk.getIndex(),	×
154	uploadRequest.getS3File().getUploadId(),	×
155	uploadRequest.getS3File().getTargetId(),	×
156	String.format(	×
157	"%s/p%s",
158	uploadRequest.getS3Filename(),	×
159	chunk.getIndex()	×
160	)
161	))).toSingle(() -> uploadProgress.addProgress(chunk.getValue().length)));	×
162	}
163
164	/**
165	* Check {@link BynderApi#getPollStatus} for more information.
166	*/
167	private Completable pollProcessing(final String importId) {
NEW 168	LOG.info("Polling status for importId: " + importId);	×
169	return getPollStatus(new PollStatusQuery(importId.split(",")))	×
170	.delay(POLLING_IDLE_TIME, TimeUnit.MILLISECONDS)	×
171	.repeat()	×
172	.take(MAX_POLLING_ITERATIONS)	×
173	.takeUntil(pollStatus -> pollStatus.processingDone(importId) \|\| pollStatus.processingFailed(importId))	×
174	.lastElement()	×
175	.toSingle()	×
176	.flatMapCompletable(pollStatus -> {	×
177	if (pollStatus.processingFailed(importId)) {	×
178	return Completable.error(new BynderUploadException("Processing media failed."));	×
179	}
180	return Completable.complete();	×
181	});
182	}
183
184	/**
185	* Calls {@link FileUploader#saveMedia(SaveMediaQuery)} to save the completely uploaded file in
186	* Bynder.
187	*
188	* @param uploadQuery Upload query with the information to upload the file.
189	* @return {@link Single} with the {@link SaveMediaResponse} information.
190	*/
191	private Single<SaveMediaResponse> saveUploadedMedia(final String importId, final UploadQuery uploadQuery) {
192	SaveMediaQuery saveMediaQuery = new SaveMediaQuery(importId)	×
193	.setAudit(uploadQuery.isAudit())	×
194	.setMetaproperties(uploadQuery.getMetaproperties());	×
195
196	if (uploadQuery.getMediaId() == null) {	×
197	// A new asset will be created for the uploaded file.
198	return saveMedia(saveMediaQuery	×
199	.setBrandId(uploadQuery.getBrandId())	×
200	.setName(uploadQuery.getAssetName())	×
201	.setTags(uploadQuery.getTags())	×
202	);
203	} else {
204	// The uploaded file will be attached to an existing asset.
205	return saveMedia(saveMediaQuery	×
206	.setMediaId(uploadQuery.getMediaId())	×
207	);
208	}
209	}
210
211	/**
212	* Check {@link BynderApi#getClosestS3Endpoint()} for more information.
213	*/
214	private Single<String> getClosestS3Endpoint() {
215	return bynderApi.getClosestS3Endpoint().singleOrError().map(RXUtils::getResponseBody);	×
216	}
217
218	/**
219	* Check {@link BynderApi#getUploadInformation(Map)} for more information.
220	*/
221	private Single<UploadRequest> getUploadInformation(final RequestUploadQuery requestUploadQuery) {
222	Map<String, String> params = queryDecoder.decode(requestUploadQuery);	×
223	return bynderApi.getUploadInformation(params).singleOrError().map(RXUtils::getResponseBody);	×
224	}
225
226	/**
227	* Check {@link BynderApi#registerChunk(Map)} for more information.
228	*/
229	private Completable registerChunk(final RegisterChunkQuery registerChunkQuery) {
230	Map<String, String> params = queryDecoder.decode(registerChunkQuery);	×
231	return bynderApi.registerChunk(params).ignoreElements();	×
232	}
233
234	/**
235	* Check {@link BynderApi#finaliseUpload(Map)} for more information.
236	*/
237	private Single<String> finaliseUpload(final FinaliseUploadQuery finaliseUploadQuery) {
238	Map<String, String> params = queryDecoder.decode(finaliseUploadQuery);	×
239	return bynderApi.finaliseUpload(params).singleOrError().map(RXUtils::getResponseBody)	×
240	.map(FinaliseResponse::getImportId);	×
241	}
242
243	private Single<UploadAdditionalMediaResponse> finaliseUploadAdditional(final FinaliseUploadAdditionalQuery finaliseUploadQuery, String mediaId) {
244	Map<String, String> params = queryDecoder.decode(finaliseUploadQuery);	×
245	return bynderApi.finaliseUploadAdditional(mediaId, params).singleOrError().map(RXUtils::getResponseBody);	×
246	}
247
248	/**
249	* Check {@link BynderApi#getPollStatus(Map)} for more information.
250	*/
251	private Single<PollStatus> getPollStatus(final PollStatusQuery pollStatusQuery) {
252	Map<String, String> params = queryDecoder.decode(pollStatusQuery);	×
NEW 253	LOG.info("Polling status for: " + params);	×
NEW 254	return bynderApi.getPollStatus(params)	×
NEW 255	.doOnNext(response -> {	×
NEW 256	if (response.raw() != null && response.raw().request() != null) {	×
NEW 257	LOG.info("Request URL for poll: " + response.raw().request().url());	×
258	}
NEW 259	})	×
NEW 260	.singleOrError()	×
NEW 261	.map(RXUtils::getResponseBody);	×
262	}
263
264	/**
265	* Check {@link BynderApi#saveMedia(Map)} for more information.
266	*/
267	private Single<SaveMediaResponse> saveMedia(final SaveMediaQuery saveMediaQuery) {
268	Map<String, String> params = queryDecoder.decode(saveMediaQuery);	×
269	return bynderApi.saveMedia(params).singleOrError().map(RXUtils::getResponseBody);	×
270	}
271	}

Bynder / bynder-java-sdk / 16054419974

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous