14820042690

Committed 04 May 2025 09:55AM UTC coverage: 63.816% (-2.1%) from 65.942%

Build # 14820042690

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #35 from mborne/34-database

(closes #34)

Run Details

67 of 86 new or added lines in 7 files covered. (77.91%)

7 existing lines in 1 file now uncovered.

194 of 304 relevant lines covered (63.82%)

0.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.38

/src/Command/FetchAllCommand.php

<?php

namespace MBO\GitManager\Command;

use Doctrine\ORM\EntityManagerInterface;
use Doctrine\Persistence\ManagerRegistry;
use Gitonomy\Git\Admin as GitAdmin;
use Gitonomy\Git\Repository as GitRepository;
use MBO\GitManager\Entity\Project;
use MBO\GitManager\Filesystem\LocalFilesystem;
use MBO\GitManager\Git\Analyzer;
use MBO\GitManager\Helpers\ProjectHelpers;
use MBO\GitManager\Repository\ProjectRepository;
use MBO\RemoteGit\ClientFactory;
use MBO\RemoteGit\ClientOptions;
use MBO\RemoteGit\Filter\FilterCollection;
use MBO\RemoteGit\Filter\IncludeRegexpFilter;
use MBO\RemoteGit\FindOptions;
use MBO\RemoteGit\ProjectInterface;
use Psr\Log\LogLevel;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Logger\ConsoleLogger;
use Symfony\Component\Console\Output\OutputInterface;

/**
 * Clone remote git projects to local directory.
 *
 * @author mborne
 */
class FetchAllCommand extends Command
{
    public function __construct(
        private ManagerRegistry $managerRegistry,
        private ProjectRepository $projectRepository,
        private EntityManagerInterface $em,
        private LocalFilesystem $localFilesystem,
        private Analyzer $analyzer,
    ) {
        parent::__construct();
    }

    protected function configure(): void
    {
        $this
            // the name of the command (the part after "bin/console")
            ->setName('git:fetch-all')

            // the short description shown while running "php bin/console list"
            ->setDescription('Fetch all repositories to local directory')
            /*
             * Git client options
             */
            ->addArgument('url', InputArgument::REQUIRED)
            ->addArgument('token')

            ->addOption('type', null, InputOption::VALUE_REQUIRED, 'Remote git type (gitlab-v4,github,gogs-v1,...)')

            ->addOption('orgs', 'o', InputOption::VALUE_REQUIRED, 'Find projects according to given organization names')
            ->addOption('users', 'u', InputOption::VALUE_REQUIRED, 'Find projects according to given user names')

            ->addOption('include', null, InputOption::VALUE_REQUIRED, 'Filter according to a given regexp, for ex : "(ansible)"')
        ;
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        $logger = $this->createLogger($output);

        $logger->info('[git:fetch-all] started...');

        /*
         * Create git client according to parameters
         */
        $clientOptions = new ClientOptions();
        $clientOptions->setUrl($input->getArgument('url'));
        $token = $input->getArgument('token');
        $clientOptions->setToken($token);

        $type = $input->getOption('type');
        if (!empty($type)) {
            $clientOptions->setType($type);
        }
        $client = ClientFactory::createClient(
            $clientOptions,
            $logger
        );

        /*
         * Create repository listing filter (git level)
         */
        $findOptions = new FindOptions();
        /* orgs option */
        $orgs = $input->getOption('orgs');
        if (!empty($orgs)) {
            $findOptions->setOrganizations(explode(',', $orgs));
        }
        /* users option */
        $users = $input->getOption('users');
        if (!empty($users)) {
            $findOptions->setUsers(explode(',', $users));
        }

        $filterCollection = new FilterCollection($logger);
        $findOptions->setFilter($filterCollection);

        /* include option */
        if (!empty($input->getOption('include'))) {
            $filterCollection->addFilter(new IncludeRegexpFilter(
                $input->getOption('include')
            ));
        }

        /*
         * Find projects
         */
        $projects = $client->find($findOptions);

        foreach ($projects as $project) {
            $logger->info(sprintf(
                '[%s] %s ...',
                $project->getName(),
                $project->getHttpUrl()
            ));
            try {
                $this->fetchOrClone($project, $token);
                $entity = $this->createOrUpdateProjectEntity($project);
                $this->em->persist($entity);
                $this->em->flush();
            } catch (\Exception $e) {
                $logger->error(sprintf(
                    '[%s] %s : "%s"',
                    $project->getName(),
                    $project->getHttpUrl(),
                    $e->getMessage()
                ));
                $this->managerRegistry->resetManager();
            }
        }

        $logger->info('[git:fetch-all] completed');

        return self::SUCCESS;
    }

    /**
     * Create or update project entity based on the given project interface.
     */
    protected function createOrUpdateProjectEntity(ProjectInterface $project): Project
    {
        $uid = ProjectHelpers::getUid($project);
        /** @var Project|null */
        $entity = $this->projectRepository->findOneBy(['id' => $uid]);
        if (null === $entity) {
            $entity = new Project();
            $entity->setId($uid);
        }
        $entity
            ->setName($project->getName())
            ->setHttpUrl($project->getHttpUrl())
            ->setDefaultBranch($project->getDefaultBranch())
            ->setArchived($project->isArchived())
            ->setVisibility($project->getVisibility()?->toString())
            ->setFullName(ProjectHelpers::getFullName($project))
        ;

        $this->analyzer->analyze($entity);

        $entity->setFetchedAt(new \DateTime('now'));

        return $entity;
    }

    protected function fetchOrClone(ProjectInterface $project, ?string $token): void
    {
        /*
         * Inject token in url to clone repository
         */
        $projectUrl = $project->getHttpUrl();
        $cloneUrl = $projectUrl;
        if (!empty($token)) {
            $scheme = parse_url($projectUrl, PHP_URL_SCHEME);
            $cloneUrl = str_replace("$scheme://", "$scheme://user-token:$token@", $projectUrl);
        }

        /*
        * fetch or clone repository to localPath
        */
        $fullName = ProjectHelpers::getFullName($project);
        $localPath = $this->localFilesystem->getRootPath().'/'.$fullName;
        if (file_exists($localPath)) {
            $gitRepository = new GitRepository($localPath);
            // use token to fetch
            $gitRepository->run('remote', [
                'set-url',
                'origin',
                $cloneUrl,
            ]);
            // update local repository
            $gitRepository->run('fetch', ['origin', '--prune', '--prune-tags']);
            // remove token
            $gitRepository->run('remote', [
                'set-url',
                'origin',
                $projectUrl,
            ]);
        } else {
            GitAdmin::cloneTo($localPath, $cloneUrl, false);
            $gitRepository = new GitRepository($localPath);
            // remove token
            $gitRepository->run('remote', [
                'set-url',
                'origin',
                $projectUrl,
            ]);
        }
    }

    /**
     * Create console logger.
     */
    protected function createLogger(OutputInterface $output): ConsoleLogger
    {
        $verbosityLevelMap = [
            LogLevel::NOTICE => OutputInterface::VERBOSITY_NORMAL,
            LogLevel::INFO => OutputInterface::VERBOSITY_NORMAL,
        ];

        return new ConsoleLogger($output, $verbosityLevelMap);
    }
}

1	<?php
2
3	namespace MBO\GitManager\Command;
4
5	use Doctrine\ORM\EntityManagerInterface;
6	use Doctrine\Persistence\ManagerRegistry;
7	use Gitonomy\Git\Admin as GitAdmin;
8	use Gitonomy\Git\Repository as GitRepository;
9	use MBO\GitManager\Entity\Project;
10	use MBO\GitManager\Filesystem\LocalFilesystem;
11	use MBO\GitManager\Git\Analyzer;
12	use MBO\GitManager\Helpers\ProjectHelpers;
13	use MBO\GitManager\Repository\ProjectRepository;
14	use MBO\RemoteGit\ClientFactory;
15	use MBO\RemoteGit\ClientOptions;
16	use MBO\RemoteGit\Filter\FilterCollection;
17	use MBO\RemoteGit\Filter\IncludeRegexpFilter;
18	use MBO\RemoteGit\FindOptions;
19	use MBO\RemoteGit\ProjectInterface;
20	use Psr\Log\LogLevel;
21	use Symfony\Component\Console\Command\Command;
22	use Symfony\Component\Console\Input\InputArgument;
23	use Symfony\Component\Console\Input\InputInterface;
24	use Symfony\Component\Console\Input\InputOption;
25	use Symfony\Component\Console\Logger\ConsoleLogger;
26	use Symfony\Component\Console\Output\OutputInterface;
27
28	/**
29	* Clone remote git projects to local directory.
30	*
31	* @author mborne
32	*/
33	class FetchAllCommand extends Command
34	{
35	public function __construct(
36	private ManagerRegistry $managerRegistry,
37	private ProjectRepository $projectRepository,
38	private EntityManagerInterface $em,
39	private LocalFilesystem $localFilesystem,
40	private Analyzer $analyzer,
41	) {
42	parent::__construct();	1✔
43	}
44
45	protected function configure(): void
46	{
47	$this	1✔
48	// the name of the command (the part after "bin/console")	1✔
49	->setName('git:fetch-all')	1✔
50
51	// the short description shown while running "php bin/console list"	1✔
52	->setDescription('Fetch all repositories to local directory')	1✔
53	/*	1✔
54	* Git client options	1✔
55	*/	1✔
56	->addArgument('url', InputArgument::REQUIRED)	1✔
57	->addArgument('token')	1✔
58
59	->addOption('type', null, InputOption::VALUE_REQUIRED, 'Remote git type (gitlab-v4,github,gogs-v1,...)')	1✔
60
61	->addOption('orgs', 'o', InputOption::VALUE_REQUIRED, 'Find projects according to given organization names')	1✔
62	->addOption('users', 'u', InputOption::VALUE_REQUIRED, 'Find projects according to given user names')	1✔
63
64	->addOption('include', null, InputOption::VALUE_REQUIRED, 'Filter according to a given regexp, for ex : "(ansible)"')	1✔
65	;	1✔
66	}
67
68	protected function execute(InputInterface $input, OutputInterface $output): int
69	{
70	$logger = $this->createLogger($output);	1✔
71
72	$logger->info('[git:fetch-all] started...');	1✔
73
74	/*
75	* Create git client according to parameters
76	*/
77	$clientOptions = new ClientOptions();	1✔
78	$clientOptions->setUrl($input->getArgument('url'));	1✔
79	$token = $input->getArgument('token');	1✔
80	$clientOptions->setToken($token);	1✔
81
82	$type = $input->getOption('type');	1✔
83	if (!empty($type)) {	1✔
84	$clientOptions->setType($type);	×
85	}
86	$client = ClientFactory::createClient(	1✔
87	$clientOptions,	1✔
88	$logger	1✔
89	);	1✔
90
91	/*
92	* Create repository listing filter (git level)
93	*/
94	$findOptions = new FindOptions();	1✔
95	/* orgs option */
96	$orgs = $input->getOption('orgs');	1✔
97	if (!empty($orgs)) {	1✔
98	$findOptions->setOrganizations(explode(',', $orgs));	×
99	}
100	/* users option */
101	$users = $input->getOption('users');	1✔
102	if (!empty($users)) {	1✔
103	$findOptions->setUsers(explode(',', $users));	1✔
104	}
105
106	$filterCollection = new FilterCollection($logger);	1✔
107	$findOptions->setFilter($filterCollection);	1✔
108
109	/* include option */
110	if (!empty($input->getOption('include'))) {	1✔
111	$filterCollection->addFilter(new IncludeRegexpFilter(	1✔
112	$input->getOption('include')	1✔
113	));	1✔
114	}
115
116	/*
117	* Find projects
118	*/
119	$projects = $client->find($findOptions);	1✔
120
121	foreach ($projects as $project) {	1✔
122	$logger->info(sprintf(	1✔
123	'[%s] %s ...',	1✔
124	$project->getName(),	1✔
125	$project->getHttpUrl()	1✔
126	));	1✔
127	try {
128	$this->fetchOrClone($project, $token);	1✔
129	$entity = $this->createOrUpdateProjectEntity($project);	1✔
130	$this->em->persist($entity);	1✔
131	$this->em->flush();	1✔
132	} catch (\Exception $e) {	×
133	$logger->error(sprintf(	×
134	'[%s] %s : "%s"',	×
135	$project->getName(),	×
136	$project->getHttpUrl(),	×
137	$e->getMessage()	×
138	));	×
NEW 139	$this->managerRegistry->resetManager();	×
140	}
141	}
142
143	$logger->info('[git:fetch-all] completed');	1✔
144
145	return self::SUCCESS;	1✔
146	}
147
148	/**
149	* Create or update project entity based on the given project interface.
150	*/
151	protected function createOrUpdateProjectEntity(ProjectInterface $project): Project
152	{
153	$uid = ProjectHelpers::getUid($project);	1✔
154	/** @var Project\|null */
155	$entity = $this->projectRepository->findOneBy(['id' => $uid]);	1✔
156	if (null === $entity) {	1✔
157	$entity = new Project();	1✔
158	$entity->setId($uid);	1✔
159	}
160	$entity	1✔
161	->setName($project->getName())	1✔
162	->setHttpUrl($project->getHttpUrl())	1✔
163	->setDefaultBranch($project->getDefaultBranch())	1✔
164	->setArchived($project->isArchived())	1✔
165	->setVisibility($project->getVisibility()?->toString())	1✔
166	->setFullName(ProjectHelpers::getFullName($project))	1✔
167	;	1✔
168
169	$this->analyzer->analyze($entity);	1✔
170
171	$entity->setFetchedAt(new \DateTime('now'));	1✔
172
173	return $entity;	1✔
174	}
175
176	protected function fetchOrClone(ProjectInterface $project, ?string $token): void
177	{
178	/*
179	* Inject token in url to clone repository
180	*/
181	$projectUrl = $project->getHttpUrl();	1✔
182	$cloneUrl = $projectUrl;	1✔
183	if (!empty($token)) {	1✔
184	$scheme = parse_url($projectUrl, PHP_URL_SCHEME);	×
185	$cloneUrl = str_replace("$scheme://", "$scheme://user-token:$token@", $projectUrl);	×
186	}
187
188	/*
189	* fetch or clone repository to localPath
190	*/
191	$fullName = ProjectHelpers::getFullName($project);	1✔
192	$localPath = $this->localFilesystem->getRootPath().'/'.$fullName;	1✔
193	if (file_exists($localPath)) {	1✔
194	$gitRepository = new GitRepository($localPath);	×
195	// use token to fetch
196	$gitRepository->run('remote', [	×
197	'set-url',	×
198	'origin',	×
199	$cloneUrl,	×
200	]);	×
201	// update local repository
202	$gitRepository->run('fetch', ['origin', '--prune', '--prune-tags']);	×
203	// remove token
204	$gitRepository->run('remote', [	×
205	'set-url',	×
206	'origin',	×
207	$projectUrl,	×
208	]);	×
209	} else {
210	GitAdmin::cloneTo($localPath, $cloneUrl, false);	1✔
211	$gitRepository = new GitRepository($localPath);	1✔
212	// remove token
213	$gitRepository->run('remote', [	1✔
214	'set-url',	1✔
215	'origin',	1✔
216	$projectUrl,	1✔
217	]);	1✔
218	}
219	}
220
221	/**
222	* Create console logger.
223	*/
224	protected function createLogger(OutputInterface $output): ConsoleLogger
225	{
226	$verbosityLevelMap = [	1✔
227	LogLevel::NOTICE => OutputInterface::VERBOSITY_NORMAL,	1✔
228	LogLevel::INFO => OutputInterface::VERBOSITY_NORMAL,	1✔
229	];	1✔
230
231	return new ConsoleLogger($output, $verbosityLevelMap);	1✔
232	}
233	}

mborne / git-manager / 14820042690

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous