DrydockWorkingCopyBlueprintImplementation.php
c3f0c0b0f735master

DrydockWorkingCopyBlueprintImplementation.php
View Options

			<?php

			final class DrydockWorkingCopyBlueprintImplementation
			extends DrydockBlueprintImplementation {

			const PHASE_SQUASHMERGE = 'squashmerge';
			const PHASE_REMOTEFETCH = 'blueprint.workingcopy.fetch.remote';
			const PHASE_MERGEFETCH = 'blueprint.workingcopy.fetch.staging';

			public function isEnabled() {
			return true;
			}

			public function getBlueprintName() {
			return pht('Working Copy');
			}

			public function getBlueprintIcon() {
			return 'fa-folder-open';
			}

			public function getDescription() {
			return pht('Allows Drydock to check out working copies of repositories.');
			}

			public function canAnyBlueprintEverAllocateResourceForLease(
			DrydockLease $lease) {
			return true;
			}

			public function canEverAllocateResourceForLease(
			DrydockBlueprint $blueprint,
			DrydockLease $lease) {
			return true;
			}

			public function canAllocateResourceForLease(
			DrydockBlueprint $blueprint,
			DrydockLease $lease) {
			$viewer = $this->getViewer();

			if ($this->shouldLimitAllocatingPoolSize($blueprint)) {
			return false;
			}

			return true;
			}

			public function canAcquireLeaseOnResource(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {

			// Don't hand out leases on working copies which have not activated, since
			// it may take an arbitrarily long time for them to acquire a host.
			if (!$resource->isActive()) {
			return false;
			}

			$need_map = $lease->getAttribute('repositories.map');
			if (!is_array($need_map)) {
			return false;
			}

			$have_map = $resource->getAttribute('repositories.map');
			if (!is_array($have_map)) {
			return false;
			}

			$have_as = ipull($have_map, 'phid');
			$need_as = ipull($need_map, 'phid');

			foreach ($need_as as $need_directory => $need_phid) {
			if (empty($have_as[$need_directory])) {
			// This resource is missing a required working copy.
			return false;
			}

			if ($have_as[$need_directory] != $need_phid) {
			// This resource has a required working copy, but it contains
			// the wrong repository.
			return false;
			}

			unset($have_as[$need_directory]);
			}

			if ($have_as && $lease->getAttribute('repositories.strict')) {
			// This resource has extra repositories, but the lease is strict about
			// which repositories are allowed to exist.
			return false;
			}

			if (!DrydockSlotLock::isLockFree($this->getLeaseSlotLock($resource))) {
			return false;
			}

			return true;
			}

			public function acquireLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {

			$lease
			->needSlotLock($this->getLeaseSlotLock($resource))
			->acquireOnResource($resource);
			}

			private function getLeaseSlotLock(DrydockResource $resource) {
			$resource_phid = $resource->getPHID();
			return "workingcopy.lease({$resource_phid})";
			}

			public function allocateResource(
			DrydockBlueprint $blueprint,
			DrydockLease $lease) {

			$resource = $this->newResourceTemplate($blueprint);

			$resource_phid = $resource->getPHID();

			$blueprint_phids = $blueprint->getFieldValue('blueprintPHIDs');

			$host_lease = $this->newLease($blueprint)
			->setResourceType('host')
			->setOwnerPHID($resource_phid)
			->setAttribute('workingcopy.resourcePHID', $resource_phid)
			->setAllowedBlueprintPHIDs($blueprint_phids);
			$resource->setAttribute('host.leasePHID', $host_lease->getPHID());

			$map = $this->getWorkingCopyRepositoryMap($lease);
			$resource->setAttribute('repositories.map', $map);

			$slot_lock = $this->getConcurrentResourceLimitSlotLock($blueprint);
			if ($slot_lock !== null) {
			$resource->needSlotLock($slot_lock);
			}

			$resource->allocateResource();

			$host_lease->queueForActivation();

			return $resource;
			}

			private function getWorkingCopyRepositoryMap(DrydockLease $lease) {
			$attribute = 'repositories.map';
			$map = $lease->getAttribute($attribute);

			// TODO: Leases should validate their attributes more formally.

			if (!is_array($map) \|\| !$map) {
			$message = array();
			if ($map === null) {
			$message[] = pht(
			'Working copy lease is missing required attribute "%s".',
			$attribute);
			} else {
			$message[] = pht(
			'Working copy lease has invalid attribute "%s".',
			$attribute);
			}

			$message[] = pht(
			'Attribute "repositories.map" should be a map of repository '.
			'specifications.');

			$message = implode("\n\n", $message);

			throw new Exception($message);
			}

			foreach ($map as $key => $value) {
			$map[$key] = array_select_keys(
			$value,
			array(
			'phid',
			));
			}

			return $map;
			}

			public function activateResource(
			DrydockBlueprint $blueprint,
			DrydockResource $resource) {

			$lease = $this->loadHostLease($resource);
			$this->requireActiveLease($lease);

			$command_type = DrydockCommandInterface::INTERFACE_TYPE;
			$interface = $lease->getInterface($command_type);

			// TODO: Make this configurable.
			$resource_id = $resource->getID();
			$root = "/var/drydock/workingcopy-{$resource_id}";

			$map = $resource->getAttribute('repositories.map');

			$futures = array();
			$repositories = $this->loadRepositories(ipull($map, 'phid'));
			foreach ($map as $directory => $spec) {
			// TODO: Validate directory isn't goofy like "/etc" or "../../lol"
			// somewhere?

			$repository = $repositories[$spec['phid']];
			$path = "{$root}/repo/{$directory}/";

			$future = $interface->getExecFuture(
			'git clone -- %s %s',
			(string)$repository->getCloneURIObject(),
			$path);

			$future->setTimeout($repository->getEffectiveCopyTimeLimit());

			$futures[$directory] = $future;
			}

			foreach (new FutureIterator($futures) as $key => $future) {
			$future->resolvex();
			}

			$resource
			->setAttribute('workingcopy.root', $root)
			->activateResource();
			}

			public function destroyResource(
			DrydockBlueprint $blueprint,
			DrydockResource $resource) {

			try {
			$lease = $this->loadHostLease($resource);
			} catch (Exception $ex) {
			// If we can't load the lease, assume we don't need to take any actions
			// to destroy it.
			return;
			}

			// Destroy the lease on the host.
			$lease->setReleaseOnDestruction(true);

			if ($lease->isActive()) {
			// Destroy the working copy on disk.
			$command_type = DrydockCommandInterface::INTERFACE_TYPE;
			$interface = $lease->getInterface($command_type);

			$root_key = 'workingcopy.root';
			$root = $resource->getAttribute($root_key);
			if (strlen($root)) {
			$interface->execx('rm -rf -- %s', $root);
			}
			}
			}

			public function getResourceName(
			DrydockBlueprint $blueprint,
			DrydockResource $resource) {
			return pht('Working Copy');
			}


			public function activateLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {

			$host_lease = $this->loadHostLease($resource);
			$command_type = DrydockCommandInterface::INTERFACE_TYPE;
			$interface = $host_lease->getInterface($command_type);

			$map = $lease->getAttribute('repositories.map');
			$root = $resource->getAttribute('workingcopy.root');

			$repositories = $this->loadRepositories(ipull($map, 'phid'));

			$default = null;
			foreach ($map as $directory => $spec) {
			$repository = $repositories[$spec['phid']];

			$interface->pushWorkingDirectory("{$root}/repo/{$directory}/");

			$cmd = array();
			$arg = array();

			$cmd[] = 'git clean -d --force';
			$cmd[] = 'git fetch';

			$commit = idx($spec, 'commit');
			$branch = idx($spec, 'branch');

			$ref = idx($spec, 'ref');

			// Reset things first, in case previous builds left anything staged or
			// dirty. Note that we don't reset to "HEAD" because that does not work
			// in empty repositories.
			$cmd[] = 'git reset --hard';

			if ($commit !== null) {
			$cmd[] = 'git checkout %s --';
			$arg[] = $commit;
			} else if ($branch !== null) {
			$cmd[] = 'git checkout %s --';
			$arg[] = $branch;

			$cmd[] = 'git reset --hard origin/%s';
			$arg[] = $branch;
			}

			$this->newExecvFuture($interface, $cmd, $arg)
			->setTimeout($repository->getEffectiveCopyTimeLimit())
			->resolvex();

			if (idx($spec, 'default')) {
			$default = $directory;
			}

			// If we're fetching a ref from a remote, do that separately so we can
			// raise a more tailored error.
			if ($ref) {
			$cmd = array();
			$arg = array();

			$ref_uri = $ref['uri'];
			$ref_ref = $ref['ref'];

			$cmd[] = 'git fetch --no-tags -- %s +%s:%s';
			$arg[] = $ref_uri;
			$arg[] = $ref_ref;
			$arg[] = $ref_ref;

			$cmd[] = 'git checkout %s --';
			$arg[] = $ref_ref;

			try {
			$this->newExecvFuture($interface, $cmd, $arg)
			->setTimeout($repository->getEffectiveCopyTimeLimit())
			->resolvex();
			} catch (CommandException $ex) {
			$display_command = csprintf(
			'git fetch %R %R',
			$ref_uri,
			$ref_ref);

			$error = DrydockCommandError::newFromCommandException($ex)
			->setPhase(self::PHASE_REMOTEFETCH)
			->setDisplayCommand($display_command);

			$lease->setAttribute(
			'workingcopy.vcs.error',
			$error->toDictionary());

			throw $ex;
			}
			}

			$merges = idx($spec, 'merges');
			if ($merges) {
			foreach ($merges as $merge) {
			$this->applyMerge($lease, $interface, $merge);
			}
			}

			$interface->popWorkingDirectory();
			}

			if ($default === null) {
			$default = head_key($map);
			}

			// TODO: Use working storage?
			$lease->setAttribute('workingcopy.default', "{$root}/repo/{$default}/");

			$lease->activateOnResource($resource);
			}

			public function didReleaseLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {
			// We leave working copies around even if there are no leases on them,
			// since the cost to maintain them is nearly zero but rebuilding them is
			// moderately expensive and it's likely that they'll be reused.
			return;
			}

			public function destroyLease(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease) {
			// When we activate a lease we just reset the working copy state and do
			// not create any new state, so we don't need to do anything special when
			// destroying a lease.
			return;
			}

			public function getType() {
			return 'working-copy';
			}

			public function getInterface(
			DrydockBlueprint $blueprint,
			DrydockResource $resource,
			DrydockLease $lease,
			$type) {

			switch ($type) {
			case DrydockCommandInterface::INTERFACE_TYPE:
			$host_lease = $this->loadHostLease($resource);
			$command_interface = $host_lease->getInterface($type);

			$path = $lease->getAttribute('workingcopy.default');
			$command_interface->pushWorkingDirectory($path);

			return $command_interface;
			}
			}

			private function loadRepositories(array $phids) {
			$viewer = $this->getViewer();

			$repositories = id(new PhabricatorRepositoryQuery())
			->setViewer($viewer)
			->withPHIDs($phids)
			->execute();
			$repositories = mpull($repositories, null, 'getPHID');

			foreach ($phids as $phid) {
			if (empty($repositories[$phid])) {
			throw new Exception(
			pht(
			'Repository PHID "%s" does not exist.',
			$phid));
			}
			}

			foreach ($repositories as $repository) {
			$repository_vcs = $repository->getVersionControlSystem();
			switch ($repository_vcs) {
			case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT:
			break;
			default:
			throw new Exception(
			pht(
			'Repository ("%s") has unsupported VCS ("%s").',
			$repository->getPHID(),
			$repository_vcs));
			}
			}

			return $repositories;
			}

			private function loadHostLease(DrydockResource $resource) {
			$viewer = $this->getViewer();

			$lease_phid = $resource->getAttribute('host.leasePHID');

			$lease = id(new DrydockLeaseQuery())
			->setViewer($viewer)
			->withPHIDs(array($lease_phid))
			->executeOne();
			if (!$lease) {
			throw new Exception(
			pht(
			'Unable to load lease ("%s").',
			$lease_phid));
			}

			return $lease;
			}

			protected function getCustomFieldSpecifications() {
			return array(
			'blueprintPHIDs' => array(
			'name' => pht('Use Blueprints'),
			'type' => 'blueprints',
			'required' => true,
			),
			);
			}

			protected function shouldUseConcurrentResourceLimit() {
			return true;
			}

			private function applyMerge(
			DrydockLease $lease,
			DrydockCommandInterface $interface,
			array $merge) {

			$src_uri = $merge['src.uri'];
			$src_ref = $merge['src.ref'];


			try {
			$interface->execx(
			'git fetch --no-tags -- %s +%s:%s',
			$src_uri,
			$src_ref,
			$src_ref);
			} catch (CommandException $ex) {
			$display_command = csprintf(
			'git fetch %R +%R:%R',
			$src_uri,
			$src_ref,
			$src_ref);

			$error = DrydockCommandError::newFromCommandException($ex)
			->setPhase(self::PHASE_MERGEFETCH)
			->setDisplayCommand($display_command);

			$lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());

			throw $ex;
			}


			// NOTE: This can never actually generate a commit because we pass
			// "--squash", but git sometimes runs code to check that a username and
			// email are configured anyway.
			$real_command = csprintf(
			'git -c user.name=%s -c user.email=%s merge --no-stat --squash -- %R',
			'drydock',
			'drydock@phabricator',
			$src_ref);

			try {
			$interface->execx('%C', $real_command);
			} catch (CommandException $ex) {
			$display_command = csprintf(
			'git merge --squash %R',
			$src_ref);

			$error = DrydockCommandError::newFromCommandException($ex)
			->setPhase(self::PHASE_SQUASHMERGE)
			->setDisplayCommand($display_command);

			$lease->setAttribute('workingcopy.vcs.error', $error->toDictionary());
			throw $ex;
			}
			}

			public function getCommandError(DrydockLease $lease) {
			return $lease->getAttribute('workingcopy.vcs.error');
			}

			private function execxv(
			DrydockCommandInterface $interface,
			array $commands,
			array $arguments) {
			return $this->newExecvFuture($interface, $commands, $arguments)->resolvex();
			}

			private function newExecvFuture(
			DrydockCommandInterface $interface,
			array $commands,
			array $arguments) {

			$commands = implode(' && ', $commands);
			$argv = array_merge(array($commands), $arguments);

			return call_user_func_array(array($interface, 'getExecFuture'), $argv);
			}

			}

Owner Packages

No Owners

phorge/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.phpc3f0c0b0f735master

DrydockWorkingCopyBlueprintImplementation.phpView Options

Owner Packages

phorge/src/applications/drydock/blueprint/DrydockWorkingCopyBlueprintImplementation.php
c3f0c0b0f735master

DrydockWorkingCopyBlueprintImplementation.php
View Options