476c8f25-8ad0-4d8b-bc39-a149b8c9164f

Committed 06 Jul 2024 03:12PM UTC coverage: 94.216% (-0.01%) from 94.226%

Build # 476c8f25-8ad0-4d8b-bc39-a149b8c9164f

Build Type

push

circleci

Committed by

edwh

Commit Message

Fix server hero calculations.

Run Details

25200 of 26747 relevant lines covered (94.22%)

31.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

46.93

/include/message/Attachment.php

<?php

namespace Freegle\Iznik;


use Jenssegers\ImageHash\ImageHash;

# TODO:
# - migrate old images across
# - retire externalurl
# - retire archiving

# This is a base class
class Attachment {
    /** @var  $dbhr LoggedPDO */
    private $dbhr;
    /** @var  $dbhm LoggedPDO */
    private $dbhm;
    private $id, $table, $hash, $archived, $externalurl, $externaluid, $externalmods;

    /**
     * @return null
     */
    public function getId() {
        return $this->id;
    }

    const TYPE_MESSAGE = 'Message';
    const TYPE_GROUP = 'Group';
    const TYPE_NEWSLETTER = 'Newsletter'; // Migrated
    const TYPE_COMMUNITY_EVENT = 'CommunityEvent';  // Migrated
    const TYPE_CHAT_MESSAGE = 'ChatMessage';
    const TYPE_USER = 'User';
    const TYPE_NEWSFEED = 'Newsfeed'; // Migrated
    const TYPE_VOLUNTEERING = 'Volunteering'; // Migrated
    const TYPE_STORY = 'Story'; // Migrated
    const TYPE_NOTICEBOARD = 'Noticeboard'; // Migrated

    /**
     * @return mixed
     */
    public function getHash() {
        return $this->hash;
    }

    public function getPath($thumb = false, $id = null, $archived = false, $mods = NULL) {
        if ($this->externaluid) {
            $u = new UploadCare();
            $mods = $mods ? $mods : $this->externalmods;
            return $u->getUrl($this->externaluid, $mods);
        }

        if ($this->externalurl) {
            return $this->externalurl;
        }

        # We serve up our attachment names as though they are files.
        # When these are fetched it will go through image.php
        $id = $id ? $id : $this->id;

        switch ($this->type) {
            case Attachment::TYPE_MESSAGE:
                $name = 'img';
                break;
            case Attachment::TYPE_GROUP:
                $name = 'gimg';
                break;
            case Attachment::TYPE_NEWSLETTER:
                $name = 'nimg';
                break;
            case Attachment::TYPE_COMMUNITY_EVENT:
                $name = 'cimg';
                break;
            case Attachment::TYPE_VOLUNTEERING:
                $name = 'oimg';
                break;
            case Attachment::TYPE_CHAT_MESSAGE:
                $name = 'mimg';
                break;
            case Attachment::TYPE_USER:
                $name = 'uimg';
                break;
            case Attachment::TYPE_NEWSFEED:
                $name = 'fimg';
                break;
            case Attachment::TYPE_STORY:
                $name = 'simg';
                break;
            case Attachment::TYPE_NOTICEBOARD:
                $name = 'bimg';
                break;
        }

        $name = $thumb ? "t$name" : $name;
        $domain = ($this->archived || $archived) ? IMAGE_ARCHIVED_DOMAIN : IMAGE_DOMAIN;

        return ("https://$domain/{$name}_$id.jpg");
    }

    public function getPublic() {
        $ret = array(
            'id' => $this->id,
            'hash' => $this->hash,
            $this->idatt => $this->{$this->idatt}
        );

        $ret['path'] = $this->getPath(false);
        $ret['paththumb'] = $this->getPath(true);
        $ret['mods'] = $this->externalmods;

        return ($ret);
    }

    function __construct(LoggedPDO $dbhr, LoggedPDO $dbhm, $id = null, $type = Attachment::TYPE_MESSAGE, $atts = null) {
        $this->dbhr = $dbhr;
        $this->dbhm = $dbhm;
        $this->id = $id;
        $this->type = $type;
        $this->archived = false;
        $url = '';
        $this->uidname = 'externaluid';
        $this->modsname = 'externalmods';
        $uid = ', externaluid';
        $mods = ', externalmods';

        switch ($type) {
            case Attachment::TYPE_MESSAGE:
            {
                $this->table = 'messages_attachments';
                $this->idatt = 'msgid';
                break;
            }
            case Attachment::TYPE_GROUP:
                $this->table = 'groups_images';
                $this->idatt = 'groupid';
                break;
            case Attachment::TYPE_NEWSLETTER:
                $this->table = 'newsletters_images';
                $this->idatt = 'articleid';
                break;
            case Attachment::TYPE_COMMUNITY_EVENT:
                $this->table = 'communityevents_images';
                $this->idatt = 'eventid';
                break;
            case Attachment::TYPE_VOLUNTEERING:
                $this->table = 'volunteering_images';
                $this->idatt = 'opportunityid';
                break;
            case Attachment::TYPE_CHAT_MESSAGE:
                $this->table = 'chat_images';
                $this->idatt = 'chatmsgid';
                break;
            case Attachment::TYPE_USER:
            {
                $this->table = 'users_images';
                $this->idatt = 'userid';
                $this->externalurlname = 'url';
                $url = ', url';
                break;
            }
            case Attachment::TYPE_NEWSFEED:
                $this->table = 'newsfeed_images';
                $this->idatt = 'newsfeedid';
                break;
            case Attachment::TYPE_STORY:
                $this->table = 'users_stories_images';
                $this->idatt = 'storyid';
                break;
            case Attachment::TYPE_NOTICEBOARD:
                $this->table = 'noticeboards_images';
                $this->idatt = 'noticeboardid';
                break;
        }

        if ($id) {
            $sql = "SELECT {$this->idatt}, hash, archived $url $uid $mods FROM {$this->table} WHERE id = ?;";
            $as = $atts ? [$atts] : $this->dbhr->preQuery($sql, [$id]);
            foreach ($as as $att) {
                $this->hash = $att['hash'];
                $this->archived = $att['archived'];
                $this->externalurl = Utils::presdef($this->externalurlname, $att, null);
                $this->externaluid = Utils::presdef($this->uidname, $att, null);
                $this->externalmods = Utils::presdef($this->modsname, $att, null);
                $this->{$this->idatt} = $att[$this->idatt];
            }
        }
    }

    public function create($id, $data, $uid = NULL, $url = null, $stripExif = TRUE, $mods = NULL) {
        if ($url) {
            # We need to fetch the data from an external URL.
            $ctx = stream_context_create(['http' =>
                [
                    'timeout' => 120
                ]
             ]);

            $data = @file_get_contents($url, false, $ctx);
        }

        if (!$uid && $data) {
            # We have the literal data.  We want to avoid uploading the same image multiple times - something
            # which is particularly likely to happen with TN because it crossposts a lot and each separate message
            # (from our p.o.v.) contains a link to the same images.  We do this by doing a perceptual hash of the
            # image and having a local dirty cache of hashes we've seen before and the corresponding Uploadcare
            # uid.  We rely on servers being rebooted before this gets too large.
            #
            # Uploadcare also creates a hash, and that's what gets stored in the DB, but there is no guarantee
            # that their hash algorithm is the same as ours, and we need the hash value precisely to avoid
            # the upload.
            $hasher = new ImageHash;
            $img = @imagecreatefromstring($data);
            $uid = NULL;
            $fn = NULL;

            if ($img) {
                $hash = $hasher->hash($img);
                $fn = "/tmp/imagehash-$hash";

                if (file_exists($fn)) {
                    $uid = file_get_contents($fn);
                    #error_log("Hash match on $hash for $id gives $uid");
                }
            }

            if (!$uid) {
                # No match - upload.
                $uc = new UploadCare();
                $uid = $uc->upload($data, 'image/jpeg');
                file_put_contents($fn, $uid);
            }
        }

        if ($uid) {
            # We now have an image on Uploadcare.
            $uc = new UploadCare();

            if ($stripExif) {
                // The uploaded photo will contain EXIF data, and there isn't currently a way to strip that out on
                // upload.  So we have to copy the image to a new one which "bakes in" the removal of the EXIF data.
                $uid = $uc->stripExif($uid);
            }

            $this->hash = $uc->getPerceptualHash($uid);

            if ($this->externalurlname) {
                $rc = $this->dbhm->preExec(
                    "INSERT INTO {$this->table} (`{$this->idatt}`, `{$this->uidname}`, `{$this->externalurlname}`, `{$this->modsname}`, `hash`) VALUES (?, ?, ?, ?, ?);",
                    [
                        $id,
                        $uid,
                        $url,
                        json_encode($mods),
                        $this->hash,
                    ]
                );
            } else {
                $rc = $this->dbhm->preExec(
                    "INSERT INTO {$this->table} (`{$this->idatt}`, `{$this->uidname}`, `{$this->modsname}`, `hash`) VALUES (?, ?, ?, ?);",
                    [
                        $id,
                        $uid,
                        json_encode($mods),
                        $this->hash,
                    ]
                );
            }

            $imgid = $rc ? $this->dbhm->lastInsertId() : null;

            if ($imgid) {
                $this->id = $imgid;
                $this->externaluid = $uid;
                $this->externalmods = $mods;
                $this->externalurl = $url;
            }

            return ([$imgid, $uid]);
        }

        return NULL;
    }

    public function getById($id) {
        $sql = "SELECT id FROM {$this->table} WHERE {$this->idatt} = ? AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL) ORDER BY id;";
        $atts = $this->dbhr->preQuery($sql, [$id]);
        $ret = [];
        foreach ($atts as $att) {
            $ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id']);
        }

        return ($ret);
    }

    public function getByIds($ids) {
        $ret = [];

        if (count($ids)) {
            $sql = "SELECT id, {$this->idatt}, hash, archived, externaluid, externalmods FROM {$this->table} 
                       WHERE {$this->idatt} IN (" . implode(',', $ids) . ") 
                       AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL) 
                       ORDER BY `primary` DESC, id;";
            #error_log($sql);
            $atts = $this->dbhr->preQuery($sql);
            foreach ($atts as $att) {
                $ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id'], $this->type, $att);
            }
        }

        return ($ret);
    }

    public function getByImageIds($ids) {
        $ret = [];
        if (count($ids)) {
            $sql = "SELECT id, {$this->idatt}, hash, archived FROM {$this->table} WHERE id IN (" . implode(
                    ',',
                    $ids
                ) . ") AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL) ORDER BY id;";
            $atts = $this->dbhr->preQuery($sql);
            foreach ($atts as $att) {
                $ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id'], $this->type, $att);
            }
        }

        return ($ret);
    }

    public function scp($host, $data, $fn, &$failed) {
        $connection = @ssh2_connect($host, 22);
        $failed = true;

        if ($connection) {
            if (@ssh2_auth_pubkey_file(
                $connection,
                CDN_SSH_USER,
                CDN_SSH_PUBLIC_KEY,
                CDN_SSH_PRIVATE_KEY
            )) {
                $temp = tempnam(sys_get_temp_dir(), "img_archive_$fn");
                file_put_contents($temp, $data);
                $rem = "/var/www/iznik/images/$fn";
                $retry = 0;

                do {
                    $rc = ssh2_scp_send($connection, $temp, $rem, 0644);

                    if (!$rc) {
                        $msg = "SCP of $rem failed, retry $retry";
                        \Sentry\captureMessage($msg);
                        error_log($msg);
                        sleep(1);
                    }

                    $retry++;
                } while (!$rc && $retry < 5);
                $failed = !$rc;
                unlink($temp);
                error_log("scp $temp to $host $rem returned $rc failed? $failed");
            }

            # Exit gracefully - might help with file truncation.
            ssh2_exec($connection, 'exit');
        }
    }

    public function archive() {
        if ($this->externalurl || $this->externaluid) {
            // We don't archive external images.
            return;
        }

        # We archive out of the DB onto our two CDN image hosts.  This reduces load on the servers because we don't
        # have to serve the images up, and it also reduces the disk space we need within the DB (which is not an ideal
        # place to store large amounts of image data);
        #
        # If we fail then we leave it unchanged for next time.
        $data = $this->getData();
        $rc = true;

        if ($data) {
            $rc = false;

            try {
                $name = null;

                # Only these types are in archive_attachments.
                switch ($this->type) {
                    case Attachment::TYPE_MESSAGE:
                        $tname = 'timg';
                        $name = 'img';
                        break;
                    case Attachment::TYPE_CHAT_MESSAGE:
                        $tname = 'tmimg';
                        $name = 'mimg';
                        break;
                    case Attachment::TYPE_NEWSFEED:
                        $tname = 'tfimg';
                        $name = 'fimg';
                        break;
                    case Attachment::TYPE_COMMUNITY_EVENT:
                        $tname = 'tcimg';
                        $name = 'cimg';
                        break;
                    case Attachment::TYPE_NOTICEBOARD:
                        $tname = 'tbimg';
                        $name = 'bimg';
                        break;
                }

                if ($name) {
                    $failed = false;

                    foreach ([CDN_HOST_1, CDN_HOST_2] as $host) {
                        # Upload the thumbnail.  If this fails we'll leave it untouched.
                        $i = new Image($data);
                        if ($i->img) {
                            $i->scale(250, 250);
                            $thumbdata = $i->getData(100);
                            $this->scp($host, $thumbdata, "{$tname}_{$this->id}.jpg", $failed2);
                            $failed |= $failed2;
                            $this->scp($host, $data, "{$name}_{$this->id}.jpg", $failed3);
                            $failed |= $failed3;
                        } else {
                            error_log("...failed to create image {$this->id}");
                        }
                    }

                    $rc = !$failed;
                }
            } catch (\Exception $e) {
                error_log("Archive failed " . $e->getMessage());
            }
        }

        if ($rc) {
            # Remove from the DB.
            $sql = "UPDATE {$this->table} SET archived = 1, data = NULL WHERE id = {$this->id};";
            $this->archived = true;
            $this->dbhm->exec($sql);
        }

        return ($rc);
    }

    public function setData($data) {
        $this->dbhm->preExec("UPDATE {$this->table} SET archived = 0, data = ? WHERE id = ?;", [
            $data,
            $this->id
        ]);
    }

    public function fgc($url, $use_include_path, $ctx) {
        return @file_get_contents($url, $use_include_path, $ctx);
    }

    public function canRedirect() {
        if ($this->externaluid) {
            $u = new UploadCare();
            return $u->getUrl($this->externaluid, $this->externalmods);
        } else if ($this->externalurl) {
            return $this->externalurl;
        } else {
            if ($this->archived) {
                # Only these types are in archive_attachments.
                switch ($this->type) {
                    case Attachment::TYPE_MESSAGE:
                        $tname = 'timg';
                        $name = 'img';
                        break;
                    case Attachment::TYPE_CHAT_MESSAGE:
                        $tname = 'tmimg';
                        $name = 'mimg';
                        break;
                    case Attachment::TYPE_NEWSFEED:
                        $tname = 'tfimg';
                        $name = 'fimg';
                        break;
                    case Attachment::TYPE_COMMUNITY_EVENT:
                        $tname = 'tcimg';
                        $name = 'cimg';
                        break;
                    case Attachment::TYPE_NOTICEBOARD:
                        $tname = 'tbimg';
                        $name = 'bimg';
                        break;
                }

                return 'https://' . IMAGE_ARCHIVED_DOMAIN . "/{$name}_{$this->id}.jpg";
            }
        }

        return false;
    }

    public function getData() {
        $ret = null;

        $url = $this->canRedirect();

        if ($url) {
            # This attachment has been archived out of our database, to a CDN.  Normally we would expect
            # that we wouldn't come through here, because we'd serve up an image link directly to the CDN, but
            # there is a timing window where we could archive after we've served up a link, so we have
            # to handle it.
            #
            # We fetch the data - not using SSL as we don't need to, and that host might not have a cert.  And
            # we put it back in the DB, because we are probably going to fetch it again.
            #
            # Apply a short timeout to avoid hanging the server if Azure is down.
            $ctx = stream_context_create(
                array(
                    'http' =>
                        array(
                            'timeout' => 2,
                        )
                )
            );

            $ret = $this->fgc($url, false, $ctx);
        } else {
            $sql = "SELECT * FROM {$this->table} WHERE id = ?;";
            $datas = $this->dbhr->preQuery($sql, [$this->id]);

            foreach ($datas as $data) {
                $ret = $data['data'];
            }
        }

        return ($ret);
    }

    public function findWebReferences() {
        # Find a web page containing this imge, if any.
        $ret = null;

        if ($this->type == Attachment::TYPE_MESSAGE) {
            $data = $this->getData();
            $base64 = base64_encode($data);

            $r_json = '{
                "requests": [
                    {
                      "image": {
                        "content":"' . $base64 . '"
                      },
                      "features": [
                          {
                            "type": "WEB_DETECTION",
                            "maxResults": 1
                          }
                      ]
                    }
                ]
            }';

            $curl = curl_init();
            curl_setopt(
                $curl,
                CURLOPT_URL,
                'https://vision.googleapis.com/v1/images:annotate?key=' . GOOGLE_VISION_KEY
            );
            curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
            curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-type: application/json"));
            curl_setopt($curl, CURLOPT_POST, true);
            curl_setopt($curl, CURLOPT_POSTFIELDS, $r_json);
            $json_response = curl_exec($curl);
            $status = curl_getinfo($curl, CURLINFO_HTTP_CODE);

            if ($status) {
                $rsp = json_decode($json_response, true);
                #error_log("Identified {$this->id} by Google $json_response for $r_json");
                error_log("Matching " . var_export($rsp, true));

                if ($rsp &&
                    array_key_exists('responses', $rsp) &&
                    count($rsp['responses']) > 0 &&
                    array_key_exists('webDetection', $rsp['responses'][0]) &&
                    array_key_exists('pagesWithMatchingImages', $rsp['responses'][0]['webDetection'])) {
                    $rsps = $rsp['responses'][0]['webDetection']['pagesWithMatchingImages'];

                    foreach ($rsps as $r) {
                        if (array_key_exists('fullMatchingImages', $r) && strpos($r['url'], USER_SITE) === false) {
                            $ret = $r['url'];
                        }
                    }
                    error_log(var_export($rsps, true));
                }
            }

            curl_close($curl);
        }

        return ($ret);
    }

    public function ocr($data = null, $returnfull = false, $video = false) {
        # Identify text in an attachment using Google Vision API.
        $base64 = $data ? $data : base64_encode($this->getData());

        if ($video) {
//            "videoContext": {
//                "textDetectionConfig": {
//                    "languageHints": ["en"]
//                }
//              }
            $r_json = '{
              "inputContent": "' . $base64 . '",
              "features": ["TEXT_DETECTION"],
            }';
        } else {
            $r_json = '{
                "requests": [
                    {
                      "image": {
                        "content":"' . $base64 . '",
                      },
                      "features": [
                          {
                            "type": "TEXT_DETECTION"
                          }
                      ],
                      "imageContext": {
                        "languageHints": [
                          "en"
                        ]
                      }
                    }
                ]
            }';
        }

        $url = 'https://vision.googleapis.com/v1/images:annotate?key=' . GOOGLE_VISION_KEY;
        $curl = curl_init();
        curl_setopt($curl, CURLOPT_URL, $url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-type: application/json"));
        curl_setopt($curl, CURLOPT_POST, true);
        curl_setopt($curl, CURLOPT_POSTFIELDS, $r_json);

        if ($video) {
            curl_setopt($curl, CURLOPT_HTTPHEADER, array("Authorization: Bearer " . GOOGLE_VIDEO_KEY));
        }

        $json_response = curl_exec($curl);
        $status = curl_getinfo($curl, CURLINFO_HTTP_CODE);

        $text = '';
        $rsps = null;

        if ($status) {
            error_log("Rsp $json_response");
            $rsp = json_decode($json_response, true);

            if ($rsp && array_key_exists('responses', $rsp) && count($rsp['responses']) > 0 && array_key_exists(
                    'textAnnotations',
                    $rsp['responses'][0]
                )) {
                $rsps = $rsp['responses'][0]['textAnnotations'];

                foreach ($rsps as $rsp) {
                    $text .= $rsp['description'] . "\n";
                    break;
                }
            }
        }

        curl_close($curl);

        return ($returnfull ? $rsps : $text);
    }

    public function setPrivate($att, $val) {
        $this->dbhm->preExec("UPDATE {$this->table} SET `$att` = ? WHERE id = {$this->id};", [$val]);
    }

    public function delete() {
        $this->dbhm->preExec("DELETE FROM {$this->table} WHERE id = {$this->id};");
    }

    public function getIdAtt() {
        return $this->idatt;
    }

    function rotate($rotate) {
        // Ensure $rotate is not negative.
        if ($this->externaluid) {
            # We can rotate this by changing external mods.
            $mods = json_decode($this->externalmods, TRUE);
            $rotate = ($rotate + 360) % 360;
            $mods['rotate'] = $rotate;
            $this->setPrivate('externalmods', json_encode($mods));
        } else {
            $data = $this->getData();
            $i = new Image($data);
            $i->rotate($rotate);
            $newdata = $i->getData(100);
            $this->setData($newdata);
        }

        if ($this->type == Attachment::TYPE_MESSAGE) {
            # Only some kinds of attachments record whether they are rotated.
            $this->recordRotate();
        }
    }

    public function recordRotate() {
        $this->setPrivate('rotated', 1);
    }
}

1	<?php
2
3	namespace Freegle\Iznik;
4
5
6	use Jenssegers\ImageHash\ImageHash;
7
8	# TODO:
9	# - migrate old images across
10	# - retire externalurl
11	# - retire archiving
12
13	# This is a base class
14	class Attachment {
15	/** @var $dbhr LoggedPDO */
16	private $dbhr;
17	/** @var $dbhm LoggedPDO */
18	private $dbhm;
19	private $id, $table, $hash, $archived, $externalurl, $externaluid, $externalmods;
20
21	/**
22	* @return null
23	*/
24	public function getId() {
25	return $this->id;	1✔
26	}
27
28	const TYPE_MESSAGE = 'Message';
29	const TYPE_GROUP = 'Group';
30	const TYPE_NEWSLETTER = 'Newsletter'; // Migrated
31	const TYPE_COMMUNITY_EVENT = 'CommunityEvent'; // Migrated
32	const TYPE_CHAT_MESSAGE = 'ChatMessage';
33	const TYPE_USER = 'User';
34	const TYPE_NEWSFEED = 'Newsfeed'; // Migrated
35	const TYPE_VOLUNTEERING = 'Volunteering'; // Migrated
36	const TYPE_STORY = 'Story'; // Migrated
37	const TYPE_NOTICEBOARD = 'Noticeboard'; // Migrated
38
39	/**
40	* @return mixed
41	*/
42	public function getHash() {
43	return $this->hash;	11✔
44	}
45
46	public function getPath($thumb = false, $id = null, $archived = false, $mods = NULL) {
47	if ($this->externaluid) {	41✔
48	$u = new UploadCare();	39✔
49	$mods = $mods ? $mods : $this->externalmods;	39✔
50	return $u->getUrl($this->externaluid, $mods);	39✔
51	}
52
53	if ($this->externalurl) {	3✔
54	return $this->externalurl;	×
55	}
56
57	# We serve up our attachment names as though they are files.
58	# When these are fetched it will go through image.php
59	$id = $id ? $id : $this->id;	3✔
60
61	switch ($this->type) {	3✔
62	case Attachment::TYPE_MESSAGE:
63	$name = 'img';	2✔
64	break;	2✔
65	case Attachment::TYPE_GROUP:
66	$name = 'gimg';	1✔
67	break;	1✔
68	case Attachment::TYPE_NEWSLETTER:
69	$name = 'nimg';	×
70	break;	×
71	case Attachment::TYPE_COMMUNITY_EVENT:
72	$name = 'cimg';	×
73	break;	×
74	case Attachment::TYPE_VOLUNTEERING:
75	$name = 'oimg';	×
76	break;	×
77	case Attachment::TYPE_CHAT_MESSAGE:
78	$name = 'mimg';	×
79	break;	×
80	case Attachment::TYPE_USER:
81	$name = 'uimg';	×
82	break;	×
83	case Attachment::TYPE_NEWSFEED:
84	$name = 'fimg';	×
85	break;	×
86	case Attachment::TYPE_STORY:
87	$name = 'simg';	×
88	break;	×
89	case Attachment::TYPE_NOTICEBOARD:
90	$name = 'bimg';	×
91	break;	×
92	}
93
94	$name = $thumb ? "t$name" : $name;	3✔
95	$domain = ($this->archived \|\| $archived) ? IMAGE_ARCHIVED_DOMAIN : IMAGE_DOMAIN;	3✔
96
97	return ("https://$domain/{$name}_$id.jpg");	3✔
98	}
99
100	public function getPublic() {
101	$ret = array(	14✔
102	'id' => $this->id,	14✔
103	'hash' => $this->hash,	14✔
104	$this->idatt => $this->{$this->idatt}	14✔
105	);	14✔
106
107	$ret['path'] = $this->getPath(false);	14✔
108	$ret['paththumb'] = $this->getPath(true);	14✔
109	$ret['mods'] = $this->externalmods;	14✔
110
111	return ($ret);	14✔
112	}
113
114	function __construct(LoggedPDO $dbhr, LoggedPDO $dbhm, $id = null, $type = Attachment::TYPE_MESSAGE, $atts = null) {
115	$this->dbhr = $dbhr;	355✔
116	$this->dbhm = $dbhm;	355✔
117	$this->id = $id;	355✔
118	$this->type = $type;	355✔
119	$this->archived = false;	355✔
120	$url = '';	355✔
121	$this->uidname = 'externaluid';	355✔
122	$this->modsname = 'externalmods';	355✔
123	$uid = ', externaluid';	355✔
124	$mods = ', externalmods';	355✔
125
126	switch ($type) {
127	case Attachment::TYPE_MESSAGE:
128	{	252✔
129	$this->table = 'messages_attachments';	252✔
130	$this->idatt = 'msgid';	252✔
131	break;	252✔
132	}	252✔
133	case Attachment::TYPE_GROUP:
134	$this->table = 'groups_images';	215✔
135	$this->idatt = 'groupid';	215✔
136	break;	215✔
137	case Attachment::TYPE_NEWSLETTER:
138	$this->table = 'newsletters_images';	1✔
139	$this->idatt = 'articleid';	1✔
140	break;	1✔
141	case Attachment::TYPE_COMMUNITY_EVENT:
142	$this->table = 'communityevents_images';	1✔
143	$this->idatt = 'eventid';	1✔
144	break;	1✔
145	case Attachment::TYPE_VOLUNTEERING:
146	$this->table = 'volunteering_images';	1✔
147	$this->idatt = 'opportunityid';	1✔
148	break;	1✔
149	case Attachment::TYPE_CHAT_MESSAGE:
150	$this->table = 'chat_images';	5✔
151	$this->idatt = 'chatmsgid';	5✔
152	break;	5✔
153	case Attachment::TYPE_USER:
154	{	1✔
155	$this->table = 'users_images';	1✔
156	$this->idatt = 'userid';	1✔
157	$this->externalurlname = 'url';	1✔
158	$url = ', url';	1✔
159	break;	1✔
160	}	1✔
161	case Attachment::TYPE_NEWSFEED:
162	$this->table = 'newsfeed_images';	1✔
163	$this->idatt = 'newsfeedid';	1✔
164	break;	1✔
165	case Attachment::TYPE_STORY:
166	$this->table = 'users_stories_images';	×
167	$this->idatt = 'storyid';	×
168	break;	×
169	case Attachment::TYPE_NOTICEBOARD:
170	$this->table = 'noticeboards_images';	1✔
171	$this->idatt = 'noticeboardid';	1✔
172	break;	1✔
173	}
174
175	if ($id) {	355✔
176	$sql = "SELECT {$this->idatt}, hash, archived $url $uid $mods FROM {$this->table} WHERE id = ?;";	31✔
177	$as = $atts ? [$atts] : $this->dbhr->preQuery($sql, [$id]);	31✔
178	foreach ($as as $att) {	31✔
179	$this->hash = $att['hash'];	31✔
180	$this->archived = $att['archived'];	31✔
181	$this->externalurl = Utils::presdef($this->externalurlname, $att, null);	31✔
182	$this->externaluid = Utils::presdef($this->uidname, $att, null);	31✔
183	$this->externalmods = Utils::presdef($this->modsname, $att, null);	31✔
184	$this->{$this->idatt} = $att[$this->idatt];	31✔
185	}
186	}
187	}
188
189	public function create($id, $data, $uid = NULL, $url = null, $stripExif = TRUE, $mods = NULL) {
190	if ($url) {	54✔
191	# We need to fetch the data from an external URL.
192	$ctx = stream_context_create(['http' =>	1✔
193	[	1✔
194	'timeout' => 120	1✔
195	]	1✔
196	]);	1✔
197
198	$data = @file_get_contents($url, false, $ctx);	1✔
199	}
200
201	if (!$uid && $data) {	54✔
202	# We have the literal data. We want to avoid uploading the same image multiple times - something
203	# which is particularly likely to happen with TN because it crossposts a lot and each separate message
204	# (from our p.o.v.) contains a link to the same images. We do this by doing a perceptual hash of the
205	# image and having a local dirty cache of hashes we've seen before and the corresponding Uploadcare
206	# uid. We rely on servers being rebooted before this gets too large.
207	#
208	# Uploadcare also creates a hash, and that's what gets stored in the DB, but there is no guarantee
209	# that their hash algorithm is the same as ours, and we need the hash value precisely to avoid
210	# the upload.
211	$hasher = new ImageHash;	52✔
212	$img = @imagecreatefromstring($data);	52✔
213	$uid = NULL;	52✔
214	$fn = NULL;	52✔
215
216	if ($img) {	52✔
217	$hash = $hasher->hash($img);	52✔
218	$fn = "/tmp/imagehash-$hash";	52✔
219
220	if (file_exists($fn)) {	52✔
221	$uid = file_get_contents($fn);	38✔
222	#error_log("Hash match on $hash for $id gives $uid");
223	}
224	}
225
226	if (!$uid) {	52✔
227	# No match - upload.
228	$uc = new UploadCare();	16✔
229	$uid = $uc->upload($data, 'image/jpeg');	16✔
230	file_put_contents($fn, $uid);	16✔
231	}
232	}
233
234	if ($uid) {	54✔
235	# We now have an image on Uploadcare.
236	$uc = new UploadCare();	54✔
237
238	if ($stripExif) {	54✔
239	// The uploaded photo will contain EXIF data, and there isn't currently a way to strip that out on
240	// upload. So we have to copy the image to a new one which "bakes in" the removal of the EXIF data.
241	$uid = $uc->stripExif($uid);	53✔
242	}
243
244	$this->hash = $uc->getPerceptualHash($uid);	54✔
245
246	if ($this->externalurlname) {	54✔
247	$rc = $this->dbhm->preExec(	×
248	"INSERT INTO {$this->table} (`{$this->idatt}`, `{$this->uidname}`, `{$this->externalurlname}`, `{$this->modsname}`, `hash`) VALUES (?, ?, ?, ?, ?);",	×
249	[	×
250	$id,	×
251	$uid,	×
252	$url,	×
253	json_encode($mods),	×
254	$this->hash,	×
255	]	×
256	);	×
257	} else {
258	$rc = $this->dbhm->preExec(	54✔
259	"INSERT INTO {$this->table} (`{$this->idatt}`, `{$this->uidname}`, `{$this->modsname}`, `hash`) VALUES (?, ?, ?, ?);",	54✔
260	[	54✔
261	$id,	54✔
262	$uid,	54✔
263	json_encode($mods),	54✔
264	$this->hash,	54✔
265	]	54✔
266	);	54✔
267	}
268
269	$imgid = $rc ? $this->dbhm->lastInsertId() : null;	54✔
270
271	if ($imgid) {	54✔
272	$this->id = $imgid;	54✔
273	$this->externaluid = $uid;	54✔
274	$this->externalmods = $mods;	54✔
275	$this->externalurl = $url;	54✔
276	}
277
278	return ([$imgid, $uid]);	54✔
279	}
280
281	return NULL;	×
282	}
283
284	public function getById($id) {
285	$sql = "SELECT id FROM {$this->table} WHERE {$this->idatt} = ? AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL) ORDER BY id;";	34✔
286	$atts = $this->dbhr->preQuery($sql, [$id]);	34✔
287	$ret = [];	34✔
288	foreach ($atts as $att) {	34✔
289	$ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id']);	6✔
290	}
291
292	return ($ret);	34✔
293	}
294
295	public function getByIds($ids) {
296	$ret = [];	105✔
297
298	if (count($ids)) {	105✔
299	$sql = "SELECT id, {$this->idatt}, hash, archived, externaluid, externalmods FROM {$this->table}	102✔
300	WHERE {$this->idatt} IN (" . implode(',', $ids) . ")	102✔
301	AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL)
302	ORDER BY `primary` DESC, id;";	102✔
303	#error_log($sql);
304	$atts = $this->dbhr->preQuery($sql);	102✔
305	foreach ($atts as $att) {	102✔
306	$ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id'], $this->type, $att);	9✔
307	}
308	}
309
310	return ($ret);	105✔
311	}
312
313	public function getByImageIds($ids) {
314	$ret = [];	2✔
315	if (count($ids)) {	2✔
316	$sql = "SELECT id, {$this->idatt}, hash, archived FROM {$this->table} WHERE id IN (" . implode(	2✔
317	',',	2✔
318	$ids	2✔
319	) . ") AND ((data IS NOT NULL AND LENGTH(data) > 0) OR archived = 1 OR externaluid IS NOT NULL) ORDER BY id;";	2✔
320	$atts = $this->dbhr->preQuery($sql);	2✔
321	foreach ($atts as $att) {	2✔
322	$ret[] = new Attachment($this->dbhr, $this->dbhm, $att['id'], $this->type, $att);	1✔
323	}
324	}
325
326	return ($ret);	2✔
327	}
328
329	public function scp($host, $data, $fn, &$failed) {
330	$connection = @ssh2_connect($host, 22);	×
331	$failed = true;	×
332
333	if ($connection) {	×
334	if (@ssh2_auth_pubkey_file(	×
335	$connection,	×
336	CDN_SSH_USER,	×
337	CDN_SSH_PUBLIC_KEY,	×
338	CDN_SSH_PRIVATE_KEY	×
339	)) {	×
340	$temp = tempnam(sys_get_temp_dir(), "img_archive_$fn");	×
341	file_put_contents($temp, $data);	×
342	$rem = "/var/www/iznik/images/$fn";	×
343	$retry = 0;	×
344
345	do {
346	$rc = ssh2_scp_send($connection, $temp, $rem, 0644);	×
347
348	if (!$rc) {	×
349	$msg = "SCP of $rem failed, retry $retry";	×
350	\Sentry\captureMessage($msg);	×
351	error_log($msg);	×
352	sleep(1);	×
353	}
354
355	$retry++;	×
356	} while (!$rc && $retry < 5);	×
357	$failed = !$rc;	×
358	unlink($temp);	×
359	error_log("scp $temp to $host $rem returned $rc failed? $failed");	×
360	}
361
362	# Exit gracefully - might help with file truncation.
363	ssh2_exec($connection, 'exit');	×
364	}
365	}
366
367	public function archive() {
368	if ($this->externalurl \|\| $this->externaluid) {	1✔
369	// We don't archive external images.
370	return;	1✔
371	}
372
373	# We archive out of the DB onto our two CDN image hosts. This reduces load on the servers because we don't
374	# have to serve the images up, and it also reduces the disk space we need within the DB (which is not an ideal
375	# place to store large amounts of image data);
376	#
377	# If we fail then we leave it unchanged for next time.
378	$data = $this->getData();	×
379	$rc = true;	×
380
381	if ($data) {	×
382	$rc = false;	×
383
384	try {
385	$name = null;	×
386
387	# Only these types are in archive_attachments.
388	switch ($this->type) {	×
389	case Attachment::TYPE_MESSAGE:
390	$tname = 'timg';	×
391	$name = 'img';	×
392	break;	×
393	case Attachment::TYPE_CHAT_MESSAGE:
394	$tname = 'tmimg';	×
395	$name = 'mimg';	×
396	break;	×
397	case Attachment::TYPE_NEWSFEED:
398	$tname = 'tfimg';	×
399	$name = 'fimg';	×
400	break;	×
401	case Attachment::TYPE_COMMUNITY_EVENT:
402	$tname = 'tcimg';	×
403	$name = 'cimg';	×
404	break;	×
405	case Attachment::TYPE_NOTICEBOARD:
406	$tname = 'tbimg';	×
407	$name = 'bimg';	×
408	break;	×
409	}
410
411	if ($name) {	×
412	$failed = false;	×
413
414	foreach ([CDN_HOST_1, CDN_HOST_2] as $host) {	×
415	# Upload the thumbnail. If this fails we'll leave it untouched.
416	$i = new Image($data);	×
417	if ($i->img) {	×
418	$i->scale(250, 250);	×
419	$thumbdata = $i->getData(100);	×
420	$this->scp($host, $thumbdata, "{$tname}_{$this->id}.jpg", $failed2);	×
421	$failed \|= $failed2;	×
422	$this->scp($host, $data, "{$name}_{$this->id}.jpg", $failed3);	×
423	$failed \|= $failed3;	×
424	} else {
425	error_log("...failed to create image {$this->id}");	×
426	}
427	}
428
429	$rc = !$failed;	×
430	}
431	} catch (\Exception $e) {	×
432	error_log("Archive failed " . $e->getMessage());	×
433	}
434	}
435
436	if ($rc) {	×
437	# Remove from the DB.
438	$sql = "UPDATE {$this->table} SET archived = 1, data = NULL WHERE id = {$this->id};";	×
439	$this->archived = true;	×
440	$this->dbhm->exec($sql);	×
441	}
442
443	return ($rc);	×
444	}
445
446	public function setData($data) {
447	$this->dbhm->preExec("UPDATE {$this->table} SET archived = 0, data = ? WHERE id = ?;", [	×
448	$data,	×
449	$this->id	×
450	]);	×
451	}
452
453	public function fgc($url, $use_include_path, $ctx) {
454	return @file_get_contents($url, $use_include_path, $ctx);	29✔
455	}
456
457	public function canRedirect() {
458	if ($this->externaluid) {	31✔
459	$u = new UploadCare();	30✔
460	return $u->getUrl($this->externaluid, $this->externalmods);	30✔
461	} else if ($this->externalurl) {	1✔
462	return $this->externalurl;	1✔
463	} else {
464	if ($this->archived) {	×
465	# Only these types are in archive_attachments.
466	switch ($this->type) {	×
467	case Attachment::TYPE_MESSAGE:
468	$tname = 'timg';	×
469	$name = 'img';	×
470	break;	×
471	case Attachment::TYPE_CHAT_MESSAGE:
472	$tname = 'tmimg';	×
473	$name = 'mimg';	×
474	break;	×
475	case Attachment::TYPE_NEWSFEED:
476	$tname = 'tfimg';	×
477	$name = 'fimg';	×
478	break;	×
479	case Attachment::TYPE_COMMUNITY_EVENT:
480	$tname = 'tcimg';	×
481	$name = 'cimg';	×
482	break;	×
483	case Attachment::TYPE_NOTICEBOARD:
484	$tname = 'tbimg';	×
485	$name = 'bimg';	×
486	break;	×
487	}
488
489	return 'https://' . IMAGE_ARCHIVED_DOMAIN . "/{$name}_{$this->id}.jpg";	×
490	}
491	}
492
493	return false;	×
494	}
495
496	public function getData() {
497	$ret = null;	29✔
498
499	$url = $this->canRedirect();	29✔
500
501	if ($url) {	29✔
502	# This attachment has been archived out of our database, to a CDN. Normally we would expect
503	# that we wouldn't come through here, because we'd serve up an image link directly to the CDN, but
504	# there is a timing window where we could archive after we've served up a link, so we have
505	# to handle it.
506	#
507	# We fetch the data - not using SSL as we don't need to, and that host might not have a cert. And
508	# we put it back in the DB, because we are probably going to fetch it again.
509	#
510	# Apply a short timeout to avoid hanging the server if Azure is down.
511	$ctx = stream_context_create(	29✔
512	array(	29✔
513	'http' =>	29✔
514	array(	29✔
515	'timeout' => 2,	29✔
516	)	29✔
517	)	29✔
518	);	29✔
519
520	$ret = $this->fgc($url, false, $ctx);	29✔
521	} else {
522	$sql = "SELECT * FROM {$this->table} WHERE id = ?;";	×
523	$datas = $this->dbhr->preQuery($sql, [$this->id]);	×
524
525	foreach ($datas as $data) {	×
526	$ret = $data['data'];	×
527	}
528	}
529
530	return ($ret);	29✔
531	}
532
533	public function findWebReferences() {
534	# Find a web page containing this imge, if any.
535	$ret = null;	×
536
537	if ($this->type == Attachment::TYPE_MESSAGE) {	×
538	$data = $this->getData();	×
539	$base64 = base64_encode($data);	×
540
541	$r_json = '{	×
542	"requests": [
543	{
544	"image": {
545	"content":"' . $base64 . '"	×
546	},
547	"features": [
548	{
549	"type": "WEB_DETECTION",
550	"maxResults": 1
551	}
552	]
553	}
554	]
555	}';	×
556
557	$curl = curl_init();	×
558	curl_setopt(	×
559	$curl,	×
560	CURLOPT_URL,	×
561	'https://vision.googleapis.com/v1/images:annotate?key=' . GOOGLE_VISION_KEY	×
562	);	×
563	curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);	×
564	curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-type: application/json"));	×
565	curl_setopt($curl, CURLOPT_POST, true);	×
566	curl_setopt($curl, CURLOPT_POSTFIELDS, $r_json);	×
567	$json_response = curl_exec($curl);	×
568	$status = curl_getinfo($curl, CURLINFO_HTTP_CODE);	×
569
570	if ($status) {	×
571	$rsp = json_decode($json_response, true);	×
572	#error_log("Identified {$this->id} by Google $json_response for $r_json");
573	error_log("Matching " . var_export($rsp, true));	×
574
575	if ($rsp &&	×
576	array_key_exists('responses', $rsp) &&	×
577	count($rsp['responses']) > 0 &&	×
578	array_key_exists('webDetection', $rsp['responses'][0]) &&	×
579	array_key_exists('pagesWithMatchingImages', $rsp['responses'][0]['webDetection'])) {	×
580	$rsps = $rsp['responses'][0]['webDetection']['pagesWithMatchingImages'];	×
581
582	foreach ($rsps as $r) {	×
583	if (array_key_exists('fullMatchingImages', $r) && strpos($r['url'], USER_SITE) === false) {	×
584	$ret = $r['url'];	×
585	}
586	}
587	error_log(var_export($rsps, true));	×
588	}
589	}
590
591	curl_close($curl);	×
592	}
593
594	return ($ret);	×
595	}
596
597	public function ocr($data = null, $returnfull = false, $video = false) {
598	# Identify text in an attachment using Google Vision API.
599	$base64 = $data ? $data : base64_encode($this->getData());	×
600
601	if ($video) {	×
602	// "videoContext": {
603	// "textDetectionConfig": {
604	// "languageHints": ["en"]
605	// }
606	// }
607	$r_json = '{	×
608	"inputContent": "' . $base64 . '",	×
609	"features": ["TEXT_DETECTION"],
610	}';	×
611	} else {
612	$r_json = '{	×
613	"requests": [
614	{
615	"image": {
616	"content":"' . $base64 . '",	×
617	},
618	"features": [
619	{
620	"type": "TEXT_DETECTION"
621	}
622	],
623	"imageContext": {
624	"languageHints": [
625	"en"
626	]
627	}
628	}
629	]
630	}';	×
631	}
632
633	$url = 'https://vision.googleapis.com/v1/images:annotate?key=' . GOOGLE_VISION_KEY;	×
634	$curl = curl_init();	×
635	curl_setopt($curl, CURLOPT_URL, $url);	×
636	curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);	×
637	curl_setopt($curl, CURLOPT_HTTPHEADER, array("Content-type: application/json"));	×
638	curl_setopt($curl, CURLOPT_POST, true);	×
639	curl_setopt($curl, CURLOPT_POSTFIELDS, $r_json);	×
640
641	if ($video) {	×
642	curl_setopt($curl, CURLOPT_HTTPHEADER, array("Authorization: Bearer " . GOOGLE_VIDEO_KEY));	×
643	}
644
645	$json_response = curl_exec($curl);	×
646	$status = curl_getinfo($curl, CURLINFO_HTTP_CODE);	×
647
648	$text = '';	×
649	$rsps = null;	×
650
651	if ($status) {	×
652	error_log("Rsp $json_response");	×
653	$rsp = json_decode($json_response, true);	×
654
655	if ($rsp && array_key_exists('responses', $rsp) && count($rsp['responses']) > 0 && array_key_exists(	×
656	'textAnnotations',	×
657	$rsp['responses'][0]	×
658	)) {	×
659	$rsps = $rsp['responses'][0]['textAnnotations'];	×
660
661	foreach ($rsps as $rsp) {	×
662	$text .= $rsp['description'] . "\n";	×
663	break;	×
664	}
665	}
666	}
667
668	curl_close($curl);	×
669
670	return ($returnfull ? $rsps : $text);	×
671	}
672
673	public function setPrivate($att, $val) {
674	$this->dbhm->preExec("UPDATE {$this->table} SET `$att` = ? WHERE id = {$this->id};", [$val]);	5✔
675	}
676
677	public function delete() {
678	$this->dbhm->preExec("DELETE FROM {$this->table} WHERE id = {$this->id};");	×
679	}
680
681	public function getIdAtt() {
682	return $this->idatt;	×
683	}
684
685	function rotate($rotate) {
686	// Ensure $rotate is not negative.
687	if ($this->externaluid) {	2✔
688	# We can rotate this by changing external mods.
689	$mods = json_decode($this->externalmods, TRUE);	2✔
690	$rotate = ($rotate + 360) % 360;	2✔
691	$mods['rotate'] = $rotate;	2✔
692	$this->setPrivate('externalmods', json_encode($mods));	2✔
693	} else {
694	$data = $this->getData();	×
695	$i = new Image($data);	×
696	$i->rotate($rotate);	×
697	$newdata = $i->getData(100);	×
698	$this->setData($newdata);	×
699	}
700
701	if ($this->type == Attachment::TYPE_MESSAGE) {	2✔
702	# Only some kinds of attachments record whether they are rotated.
703	$this->recordRotate();	2✔
704	}
705	}
706
707	public function recordRotate() {
708	$this->setPrivate('rotated', 1);	2✔
709	}
710	}

Freegle / iznik-server / 476c8f25-8ad0-4d8b-bc39-a149b8c9164f

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous