1802

pending completion

Build # 1802

Build Type

Pull #1850

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/RipMeApp/ripme/commit/<a class=hub.com/RipMeApp/ripme/commit/<a class="double-link" href="https://git"><a class=hub.com/RipMeApp/ripme/commit/<a class="double-link" href="https://git"><a class=hub.com/RipMeApp/ripme/commit/<a class="double-link" href="https://git"><a class=hub.com/RipMeApp/ripme/commit/dbe6eb19bb5dbb194d8b7774583bad72b15d3662">dbe6eb19b">&lt;a href=&quot;https://github.com/RipMeApp/ripme/commit/</a><a class="double-link" href="https://github.com/RipMeApp/ripme/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/RipMeApp/ripme/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/RipMeApp/ripme/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/RipMeApp/ripme/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/RipMeApp/ripme/commit/dbe6eb19bb5dbb194d8b7774583bad72b15d3662">dbe6eb19b</a><a href="https://github.com/RipMeApp/ripme/commit/dbe6eb19bb5dbb194d8b7774583bad72b15d3662">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/RipMeApp/ripme/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/RipMeApp/ripme/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/RipMeApp/ripme/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/RipMeApp/ripme/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/RipMeApp/ripme/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/RipMeApp/ripme/commit/dbe6eb19bb5dbb194d8b7774583bad72b15d3662&quot;&gt;dbe6eb19b&lt;/a&gt;&lt;a href=&quot;https://github.com/RipMeApp/ripme/commit/dbe6eb19bb5dbb194d8b7774583bad72b15d3662&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/RipMeApp/ripme/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/RipMeApp/ripme/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=... (continued)

Pull Request Pull Request #1850: Feature/testall

Run Details

315 of 315 new or added lines in 10 files covered. (100.0%)

3187 of 11191 relevant lines covered (28.48%)

0.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.15

/src/main/java/com/rarchives/ripme/ripper/rippers/EromeRipper.java

package com.rarchives.ripme.ripper.rippers;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.rarchives.ripme.utils.Utils;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.utils.Http;

/**
 *
 * @author losipher
 */
public class EromeRipper extends AbstractHTMLRipper {

    boolean rippingProfile;
    private HashMap<String, String> cookies = new HashMap<>();

    public EromeRipper (URL url) throws IOException {
        super(url);
    }

    @Override
    public String getDomain() {
        return "erome.com";
    }

    @Override
    public String getHost() {
        return "erome";
    }

    @Override
    public void downloadURL(URL url, int index) {
        addURLToDownload(url, getPrefix(index));
    }

    @Override
    public boolean hasQueueSupport() {
        return true;
    }

    @Override
    public boolean pageContainsAlbums(URL url) {
        Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_\\-?=]*)/?");
        Matcher ma = pa.matcher(url.toExternalForm());
        return ma.matches();
    }

    @Override
    public List<String> getAlbumsToQueue(Document doc) {
        List<String> urlsToAddToQueue = new ArrayList<>();
        for (Element elem : doc.select("div#albums > div.album > a")) {
            urlsToAddToQueue.add(elem.attr("href"));
        }
        return urlsToAddToQueue;
    }

    @Override
    public String getAlbumTitle(URL url) throws MalformedURLException {
        try {
            // Attempt to use album title as GID
            Element titleElement = getFirstPage().select("meta[property=og:title]").first();
            String title = titleElement.attr("content");
            title = title.substring(title.lastIndexOf('/') + 1);
            return getHost() + "_" + getGID(url) + "_" + title.trim();
        } catch (IOException e) {
            // Fall back to default album naming convention
            LOGGER.info("Unable to find title at " + url);
        } catch (NullPointerException e) {
            return getHost() + "_" + getGID(url);
        }
        return super.getAlbumTitle(url);
    }

    @Override
    public URL sanitizeURL(URL url) throws MalformedURLException {
        return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com"));
    }


    @Override
    public List<String> getURLsFromPage(Document doc) {
        List<String> URLs = new ArrayList<>();
        return getMediaFromPage(doc);
    }

    @Override
    public Document getFirstPage() throws IOException {
        this.setAuthCookie();
        Response resp = Http.url(this.url)
                .cookies(cookies)
                .ignoreContentType()
                .response();

        return resp.parse();
    }

    @Override
    public String getGID(URL url) throws MalformedURLException {
        Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$");
        Matcher m = p.matcher(url.toExternalForm());
        if (m.matches()) {
            return m.group(1);
        }

        p = Pattern.compile("^https?://www.erome.com/([a-zA-Z0-9_\\-?=]+)/?$");
        m = p.matcher(url.toExternalForm());

        if (m.matches()) {
            return m.group(1);
        }

        throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album");
    }

    private List<String> getMediaFromPage(Document doc) {
        List<String> results = new ArrayList<>();
        for (Element el : doc.select("img.img-front")) {
            if (el.hasAttr("src")) {
                if (el.attr("src").startsWith("https:")) {
                    results.add(el.attr("src"));
                } else {
                    results.add("https:" + el.attr("src"));
                }
            } else if (el.hasAttr("data-src")) {
                //to add images that are not loaded( as all images are lasyloaded as we scroll).
                results.add(el.attr("data-src"));
            }

        }
        for (Element el : doc.select("source[label=HD]")) {
            if (el.attr("src").startsWith("https:")) {
                results.add(el.attr("src"));
            }
            else {
                results.add("https:" + el.attr("src"));
            }
        }
        for (Element el : doc.select("source[label=SD]")) {
            if (el.attr("src").startsWith("https:")) {
                results.add(el.attr("src"));
            }
            else {
                results.add("https:" + el.attr("src"));
            }
        }

        if (results.size() == 0) {
            if (cookies.isEmpty()) {
                LOGGER.warn("You might try setting erome.laravel_session manually " +
                        "if you think this page definitely contains media.");
            }
        }

        return results;
    }

    private void setAuthCookie() {
        String sessionId = Utils.getConfigString("erome.laravel_session", null);
        if (sessionId != null) {
            cookies.put("laravel_session", sessionId);
        }
    }

}

1	package com.rarchives.ripme.ripper.rippers;
2
3	import java.io.IOException;
4	import java.net.MalformedURLException;
5	import java.net.URL;
6	import java.util.ArrayList;
7	import java.util.HashMap;
8	import java.util.List;
9	import java.util.regex.Matcher;
10	import java.util.regex.Pattern;
11
12	import com.rarchives.ripme.utils.Utils;
13	import org.jsoup.Connection.Response;
14	import org.jsoup.nodes.Document;
15	import org.jsoup.nodes.Element;
16	import org.jsoup.select.Elements;
17
18	import com.rarchives.ripme.ripper.AbstractHTMLRipper;
19	import com.rarchives.ripme.utils.Http;
20
21	/**
22	*
23	* @author losipher
24	*/
25	public class EromeRipper extends AbstractHTMLRipper {
26
27	boolean rippingProfile;
28	private HashMap<String, String> cookies = new HashMap<>();	1✔
29
30	public EromeRipper (URL url) throws IOException {
31	super(url);	1✔
32	}	1✔
33
34	@Override
35	public String getDomain() {
36	return "erome.com";	1✔
37	}
38
39	@Override
40	public String getHost() {
41	return "erome";	×
42	}
43
44	@Override
45	public void downloadURL(URL url, int index) {
46	addURLToDownload(url, getPrefix(index));	×
47	}	×
48
49	@Override
50	public boolean hasQueueSupport() {
51	return true;	×
52	}
53
54	@Override
55	public boolean pageContainsAlbums(URL url) {
56	Pattern pa = Pattern.compile("https?://www.erome.com/([a-zA-Z0-9_\\-?=]*)/?");	1✔
57	Matcher ma = pa.matcher(url.toExternalForm());	1✔
58	return ma.matches();	1✔
59	}
60
61	@Override
62	public List<String> getAlbumsToQueue(Document doc) {
63	List<String> urlsToAddToQueue = new ArrayList<>();	1✔
64	for (Element elem : doc.select("div#albums > div.album > a")) {	1✔
65	urlsToAddToQueue.add(elem.attr("href"));	1✔
66	}	1✔
67	return urlsToAddToQueue;	1✔
68	}
69
70	@Override
71	public String getAlbumTitle(URL url) throws MalformedURLException {
72	try {
73	// Attempt to use album title as GID
74	Element titleElement = getFirstPage().select("meta[property=og:title]").first();	×
75	String title = titleElement.attr("content");	×
76	title = title.substring(title.lastIndexOf('/') + 1);	×
77	return getHost() + "_" + getGID(url) + "_" + title.trim();	×
78	} catch (IOException e) {	×
79	// Fall back to default album naming convention
80	LOGGER.info("Unable to find title at " + url);	×
81	} catch (NullPointerException e) {	×
82	return getHost() + "_" + getGID(url);	×
83	}	×
84	return super.getAlbumTitle(url);	×
85	}
86
87	@Override
88	public URL sanitizeURL(URL url) throws MalformedURLException {
89	return new URL(url.toExternalForm().replaceAll("https?://erome.com", "https://www.erome.com"));	1✔
90	}
91
92
93	@Override
94	public List<String> getURLsFromPage(Document doc) {
95	List<String> URLs = new ArrayList<>();	1✔
96	return getMediaFromPage(doc);	1✔
97	}
98
99	@Override
100	public Document getFirstPage() throws IOException {
101	this.setAuthCookie();	1✔
102	Response resp = Http.url(this.url)	1✔
103	.cookies(cookies)	1✔
104	.ignoreContentType()	1✔
105	.response();	1✔
106
107	return resp.parse();	1✔
108	}
109
110	@Override
111	public String getGID(URL url) throws MalformedURLException {
112	Pattern p = Pattern.compile("^https?://www.erome.com/[ai]/([a-zA-Z0-9]*)/?$");	1✔
113	Matcher m = p.matcher(url.toExternalForm());	1✔
114	if (m.matches()) {	1✔
115	return m.group(1);	1✔
116	}
117
118	p = Pattern.compile("^https?://www.erome.com/([a-zA-Z0-9_\\-?=]+)/?$");	1✔
119	m = p.matcher(url.toExternalForm());	1✔
120
121	if (m.matches()) {	1✔
122	return m.group(1);	1✔
123	}
124
125	throw new MalformedURLException("erome album not found in " + url + ", expected https://www.erome.com/album");	×
126	}
127
128	private List<String> getMediaFromPage(Document doc) {
129	List<String> results = new ArrayList<>();	1✔
130	for (Element el : doc.select("img.img-front")) {	1✔
131	if (el.hasAttr("src")) {	1✔
132	if (el.attr("src").startsWith("https:")) {	×
133	results.add(el.attr("src"));	×
134	} else {
135	results.add("https:" + el.attr("src"));	×
136	}
137	} else if (el.hasAttr("data-src")) {	1✔
138	//to add images that are not loaded( as all images are lasyloaded as we scroll).
139	results.add(el.attr("data-src"));	1✔
140	}
141
142	}	1✔
143	for (Element el : doc.select("source[label=HD]")) {	1✔
144	if (el.attr("src").startsWith("https:")) {	×
145	results.add(el.attr("src"));	×
146	}
147	else {
148	results.add("https:" + el.attr("src"));	×
149	}
150	}	×
151	for (Element el : doc.select("source[label=SD]")) {	1✔
152	if (el.attr("src").startsWith("https:")) {	×
153	results.add(el.attr("src"));	×
154	}
155	else {
156	results.add("https:" + el.attr("src"));	×
157	}
158	}	×
159
160	if (results.size() == 0) {	1✔
161	if (cookies.isEmpty()) {	×
162	LOGGER.warn("You might try setting erome.laravel_session manually " +	×
163	"if you think this page definitely contains media.");
164	}
165	}
166
167	return results;	1✔
168	}
169
170	private void setAuthCookie() {
171	String sessionId = Utils.getConfigString("erome.laravel_session", null);	1✔
172	if (sessionId != null) {	1✔
173	cookies.put("laravel_session", sessionId);	×
174	}
175	}	1✔
176
177	}

RipMeApp / ripme / 1802

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous