Introduced Resource Repositories.

Works but requires manual `import`.
Removed `id` from resources.

Change-Id: I36ef63a5003ea39eb88d9076afd61215584916d9

bootstrap/playbooks/solar.yaml

@@ -41,3 +41,9 @@
         line: "solar_db: riak://10.0.0.2:8087"
         state: present
         create: yes
+
+##  will be uncommented when we will change embedded resources structure
+# - hosts: all
+#   tasks:
+#     - file: path=/var/lib/solar/repositories state=directory
+#     - file: src=/vagrant/resources dest=/var/lib/solar/repositories/resources state=link

doc/source/glossary.rst

@@ -44,6 +44,13 @@ Tag
 Used to create arbitrary groups of resources, later this groups will be
 used for different user operations.
 
+.. _res-repository-term:
+
+Resource Repository
+-------------------
+
+It is a named location where different :ref:`resource-term` are located.
+
 .. _res-handler-term:
 
 Handler
@@ -62,6 +69,7 @@ Used in handlers to communicate with hosts managed by Solar.
 
    :ref:`More details about transports <transports_details>`
 
+
 .. _location-id-term:
 
 location_id

requirements.txt

@@ -27,3 +27,7 @@ wrapt
 peewee
 # if you want to use lua computable inputs
 # lupa
+
+
+# if you want to use complex version check in repositories
+# semver

resources/README.md

@@ -0,0 +1,5 @@
+We're moving to repositories.
+
+To use import this dir with `solar repository import ./resources --name resources`.
+
+To update `solar repository update --overwrite ./resources --name resources`

resources/ansible_local/meta.yaml

@@ -1,4 +1,3 @@
-id: ansible_sample
 handler: ansible_playbook
 version: 0.0.1
 input:

resources/ansible_remote/meta.yaml

@@ -1,4 +1,3 @@
-id: ansible_sample
 handler: ansible_playbook
 version: 0.0.1
 input:

resources/apache_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: apache_puppet
 handler: puppet
 puppet_module: apache
 version: 1.0.0

resources/apt_repo/meta.yaml

@@ -1,4 +1,3 @@
-id: apt_repo
 handler: ansible
 version: 1.0.0
 input:

resources/ceph_keys/meta.yaml

@@ -1,4 +1,3 @@
-id: ceph_keys
 handler: shell
 version: 1.0.0
 input:

resources/ceph_mon/meta.yaml

@@ -1,4 +1,3 @@
-id: ceph_mon
 handler: puppetv2
 version: 1.0.0
 input:

resources/cinder_api_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: cinder_api_puppet
 handler: puppet
 puppet_module: cinder
 version: 1.0.0

resources/cinder_glance_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: cinder_glance_puppet
 handler: puppet
 puppet_module: cinder
 version: 1.0.0

resources/cinder_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: cinder_puppet
 handler: puppet
 actions:
     run: run.pp

resources/cinder_scheduler_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: cinder_scheduler_puppet
 handler: puppet
 puppet_module: cinder
 version: 1.0.0

resources/cinder_volume_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: cinder_volume_puppet
 handler: puppet
 puppet_module: cinder
 version: 1.0.0

resources/container_networks/meta.yaml

@@ -1,4 +1,3 @@
-id: container_networks
 handler: ansible_playbook
 version: 1.0.0
 actions:

resources/data_container/meta.yaml

@@ -1,4 +1,3 @@
-id: data_container
 handler: ansible
 version: 1.0.0
 input:

resources/dnsmasq/meta.yaml

@@ -1,4 +1,3 @@
-id: dnsmasq
 handler: ansible
 version: 1.0.0
 

resources/docker/meta.yaml

@@ -1,4 +1,3 @@
-id: docker
 handler: ansible
 version: 1.0.0
 

resources/docker_container/meta.yaml

@@ -1,4 +1,3 @@
-id: container
 handler: ansible
 version: 1.0.0
 input:

resources/ex_managed/meta.yaml

@@ -1,4 +1,3 @@
-id: managed
 handler: none
 version: 1.0.0
 managers:

resources/file/meta.yaml

@@ -1,4 +1,3 @@
-id: file
 handler: shell
 version: 1.0.0
 input:

resources/fuel_library/meta.yaml

@@ -1,4 +1,3 @@
-id: fuel_library
 handler: shell
 version: 1.0.0
 input:

resources/glance_api_service/meta.yaml

@@ -1,4 +1,3 @@
-id: container
 handler: ansible
 version: 1.0.0
 input:

resources/glance_config/meta.yaml

@@ -1,4 +1,3 @@
-id: glance_config
 handler: ansible
 version: 1.0.0
 input:

resources/glance_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: glance_puppet
 handler: puppet
 puppet_module: glance
 version: 1.0.0

resources/glance_registry_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: glance_registry_puppet
 handler: puppet
 puppet_module: glance
 version: 1.0.0

resources/glance_registry_service/meta.yaml

@@ -1,4 +1,3 @@
-id: container
 handler: ansible
 version: 1.0.0
 input:

resources/haproxy_config/meta.yaml

@@ -1,4 +1,3 @@
-id: haproxy_config
 handler: ansible
 version: 1.0.0
 input:

resources/haproxy_service/meta.yaml

@@ -1,4 +1,3 @@
-id: haproxy_service
 handler: ansible
 version: 1.0.0
 input:

resources/haproxy_service_config/meta.yaml

@@ -1,4 +1,3 @@
-id: haproxy_service_config
 handler: none
 version: 1.0.0
 input:

resources/hosts_file/meta.yaml

@@ -1,4 +1,3 @@
-id: hosts_file
 handler: ansible
 version: 1.0.0
 

resources/keystone_config/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_config
 handler: ansible
 version: 1.0.0
 

resources/keystone_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_puppet
 handler: puppet
 puppet_module: keystone
 version: 1.0.0

resources/keystone_role/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_role
 handler: ansible
 version: 1.0.0
 input:

resources/keystone_service/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_service
 handler: ansible
 version: 1.0.0
 input:

resources/keystone_service_endpoint/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_service_endpoint
 handler: ansible
 version: 1.0.0
 input:

resources/keystone_tenant/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_tenant
 handler: ansible
 version: 1.0.0
 input:

resources/keystone_user/meta.yaml

@@ -1,4 +1,3 @@
-id: keystone_user
 handler: ansible
 version: 1.0.0
 input:

resources/librarian/meta.yaml

@@ -1,4 +1,3 @@
-id: librarian
 handler: ansible
 version: 0.0.1
 actions:

resources/lxc_container/meta.yaml

@@ -1,4 +1,3 @@
-id: lxc_container
 handler: ansible_playbook
 version: 1.0.0
 actions:

resources/lxc_host/meta.yaml

@@ -1,4 +1,3 @@
-id: lxc_host
 handler: ansible_playbook
 version: 1.0.0
 actions:

resources/managed_apt/meta.yaml

@@ -1,5 +1,4 @@
 # This resource will clean
-id: apt_repo_manager
 handler: ansible
 version: 1.0.0
 input:

resources/mariadb_db/meta.yaml

@@ -1,4 +1,3 @@
-id: mariadb_db
 handler: ansible
 version: 1.0.0
 actions:

resources/mariadb_service/meta.yaml

@@ -1,4 +1,3 @@
-id: mariadb_service
 handler: ansible
 version: 1.0.0
 actions:

resources/mariadb_user/meta.yaml

@@ -1,4 +1,3 @@
-id: mariadb_user
 handler: ansible
 version: 1.0.0
 actions:

resources/neutron_agents_dhcp_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_agents_dhcp_puppet'
 input:
   ip:
     schema: str!

resources/neutron_agents_l3_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_agents_l3_puppet'
 input:
   ip:
     schema: str!

resources/neutron_agents_metadata_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_agents_metadata_puppet'
 input:
   ip:
     schema: str!

resources/neutron_agents_ml2_ovs_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_agents_ml2_ovs_puppet'
 input:
   ip:
     schema: str!

resources/neutron_plugins_ml2_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_plugins_ml2_puppet'
 input:
   ip:
     schema: str!

resources/neutron_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_puppet'
 input:
   ip:
     schema: str!

resources/neutron_server_puppet/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'neutron_server_puppet'
 actions:
     run: run.pp
     update: run.pp

resources/node_network_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: node_network_puppet
 handler: puppet
 puppet_module: l23network
 version: 1.0.0

resources/not_provisioned_node/meta.yaml

@@ -1,4 +1,3 @@
-id: not_provisioned_node
 handler: shell
 version: 1.0.0
 

resources/nova_api_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_api
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_compute_libvirt_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_compute_libvirt
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_compute_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_compute
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_conductor_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_conductor
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_generic_service_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_generic_service
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_neutron_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_neutron
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/nova_puppet/meta.yaml

@@ -1,4 +1,3 @@
-id: nova
 handler: puppet
 puppet_module: nova
 version: 1.0.0

resources/openrc_file/meta.yaml

@@ -1,4 +1,3 @@
-id: nova_config
 handler: ansible
 version: 1.0.0
 

resources/rabbitmq_config/meta.yaml

@@ -1,4 +1,3 @@
-id: rabbitmq_config
 handler: ansible
 version: 1.0.0
 input:

resources/rabbitmq_service/meta.yaml

@@ -1,5 +1,4 @@
 handler: puppet
-id: 'rabbitmq'
 input:
   ip:
     schema: str!

resources/rabbitmq_user/meta.yaml

@@ -1,4 +1,3 @@
-id: rabbitmq_user
 handler: ansible
 version: 1.0.0
 input:

resources/rabbitmq_vhost/meta.yaml

@@ -1,4 +1,3 @@
-id: rabbitmq_vhost
 handler: ansible
 version: 1.0.0
 input:

resources/remote_file/meta.yaml

@@ -1,4 +1,3 @@
-id: remote_file
 handler: shell
 version: 1.0.0
 input:

resources/riak_join_single/meta.yaml

@@ -1,4 +1,3 @@
-id: riak_join_single
 handler: ansible
 version: 1.0.0
 actions:

resources/riak_node/meta.yaml

@@ -1,4 +1,3 @@
-id: riak_node
 handler: ansible
 version: 1.0.0
 actions:

resources/ro_node/meta.yaml

@@ -1,4 +1,3 @@
-id: ro_node
 handler: none
 version: 1.0.0
 actions:

resources/solar_bootstrap/meta.yaml

@@ -1,5 +1,4 @@
 handler: ansible
-id: 'solar_bootstrap'
 input:
   ip:
     schema: str!

resources/sources/meta.yaml

@@ -1,4 +1,3 @@
-id: sources
 handler: naive_sync
 version: 1.0.0
 input:

resources/ssh_key/meta.yaml

@@ -1,4 +1,3 @@
-id: ssh_key
 handler: ansible_playbook
 version: 1.0.0
 actions:

resources/transport_rsync/meta.yaml

@@ -1,4 +1,3 @@
-id: transport_rsync
 input:
   key:
     schema: str!

resources/transport_solar_agent/meta.yaml

@@ -1,4 +1,3 @@
-id: transport_solar_agent
 handler: ansible
 input:
   solar_agent_user:

resources/transport_ssh/meta.yaml

@@ -1,4 +1,3 @@
-id: transport_ssh
 input:
   ssh_key:
     schema: str!

resources/transport_torrent/meta.yaml

@@ -1,4 +1,3 @@
-id: transport_torrent
 handler: ansible
 input:
   trackers:

resources/transports/meta.yaml

@@ -1,4 +1,3 @@
-id: transports
 input:
   transports:
     schema: [{user: str, password: str, port: int!, key: str, name: str!, trackers: [str]}]

resources/volume_group/meta.yaml

@@ -1,4 +1,3 @@
-id: volume_group
 handler: ansible
 version: 1.0.0
 input:

resources/vxlan_mesh/meta.yaml

@@ -1,4 +1,3 @@
-id: vxlan_mesh
 handler: ansible_playbook
 version: 1.0.0
 actions:

solar/cli/main.py

@@ -30,6 +30,7 @@ from solar.core import signals
 from solar.cli import base
 from solar.cli.events import events
 from solar.cli.orch import orchestration
+from solar.cli.repository import repository as cli_repository
 from solar.cli.resource import resource as cli_resource
 from solar.cli.system_log import changes
 
@@ -161,6 +162,7 @@ def run():
     main.add_command(orchestration)
     main.add_command(changes)
     main.add_command(events)
+    main.add_command(cli_repository)
     main()
 
 

solar/cli/repository.py

@@ -0,0 +1,124 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import click
+import os
+import yaml
+
+from solar.core.resource.repository import Repository
+from solar.core.resource.repository import RepositoryExists
+
+
+@click.group(help="Manages Solar repositories")
+def repository():
+    pass
+
+
+@repository.command(help="Shows all added repositories, "
+                    "or content of repository when `-r` given")
+@click.option('--repository', '-r', default=None)
+def show(repository):
+    if not repository:
+        repos = Repository.list_repos()
+        str_repos = '\n'.join(sorted(repos))
+        click.echo(str_repos)
+    else:
+        repo = Repository(repository)
+        content = yaml.safe_dump(dict(repo.get_contents()),
+                                 default_flow_style=False)
+        click.echo_via_pager(content)
+
+
+@repository.command(name='import', help="Imports repository to Solar")
+@click.argument('source', type=click.Path(exists=True, resolve_path=True))
+@click.option('--name', '-n', default=None)
+@click.option('--link', '-l', is_flag=True, default=False)
+def _import(name, source, link):
+    if name is None:
+        name = os.path.split(source)[-1]
+    repo = Repository(name)
+    try:
+        repo.create(source, link)
+    except RepositoryExists as e:
+        click.echo(click.style(str(e), fg='red'))
+    else:
+        cnt = len(list(repo.iter_contents()))
+        click.echo(
+            "Created new repository with {} resources".format(cnt))
+
+
+@repository.command(help="Updates existing repository with new content")
+@click.argument('name')
+@click.argument('source', type=click.Path(exists=True, resolve_path=True))
+@click.option('--overwrite', is_flag=True, default=False, help="If resource "
+              "already exists then overwrite contents.")
+def update(name, source, overwrite):
+    repo = Repository(name)
+    prev = len(list(repo.iter_contents()))
+    repo.update(source, overwrite)
+    now = len(list(repo.iter_contents()))
+    diff = now - prev
+    click.echo(
+        "Updated repository, with {} resources".format(diff))
+
+
+@repository.command(help="Adds new resource to repository")
+@click.argument('name')
+@click.argument('source', type=click.Path(exists=True, resolve_path=True))
+@click.option('--overwrite', is_flag=True, default=False, help="If resource "
+              "already exists then overwrite contents.")
+@click.option('--resource_name', type=str, default=None, help="Set different "
+              "name than last part of path.")
+def add(name, source, overwrite, resource_name):
+    repo = Repository(name)
+    if resource_name is None:
+        resource_name = os.path.split(source)[-1]
+    repo.add_single(name=resource_name,
+                    source=source,
+                    overwrite=overwrite)
+
+
+@repository.command(help="Destroys repository")
+@click.argument('name')
+def destroy(name):
+    repo = Repository(name)
+    repo.remove()
+
+
+@repository.command(help="Removes `spec` from Solar repositories")
+@click.argument('spec')
+def remove(spec):
+    repo, spec = Repository.parse(spec)
+    repo.remove_single(spec)
+
+
+@repository.command(help="Checks if `spec` is in Solar repositories")
+@click.argument('spec')
+@click.option('--bool', is_flag=True, default=False)
+def contains(spec, bool):
+    repo, spec = Repository.parse(spec)
+    if bool:
+        result = Repository.contains(spec)
+        if result:
+            click.echo(click.style("Exists", fg='green'))
+            return
+    else:
+        result_version = Repository.what_version(spec)
+        if result_version:
+            click.echo(click.style("Found: {}".format(result_version),
+                                   fg='green'))
+            return
+    spec_data = yaml.safe_dump(spec, default_flow_style=False)
+    click.echo(click.style("Not found: \n{}".format(spec_data),
+                           fg='red'))

solar/core/resource/repository.py

@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from collections import defaultdict
+
+import errno
+import os
+import shutil
+
+from solar import utils
+
+try:
+    import semver
+except ImportError:
+    _semver = False
+else:
+    _semver = True
+
+
+class RepositoryException(Exception):
+    pass
+
+
+class ResourceNotFound(RepositoryException):
+
+    def __init__(self, spec):
+        self.message = 'Resource definition %r not found' % spec
+
+    def __str__(self):
+        return str(self.message)
+
+
+def read_meta(base_path):
+    base_meta_file = os.path.join(base_path, 'meta.yaml')
+
+    metadata = utils.yaml_load(base_meta_file)
+    metadata.setdefault('version', '1.0.0')
+    metadata['base_path'] = os.path.abspath(base_path)
+    actions_path = os.path.join(metadata['base_path'], 'actions')
+    metadata['actions_path'] = actions_path
+    metadata['base_name'] = os.path.split(metadata['base_path'])[-1]
+
+    return metadata
+
+
+class RepositoryExists(RepositoryException):
+    pass
+
+
+class Repository(object):
+
+    db_obj = None
+    _REPOS_LOCATION = '/var/lib/solar/repositories'
+
+    def __init__(self, name):
+        self.name = name
+        # TODO: (jnowak) sanitize name
+        self.fpath = self.repo_path(self.name)
+
+    def _list_source_contents(self, source):
+        for pth in os.listdir(source):
+            single_path = os.path.join(source, pth)
+            if os.path.exists(os.path.join(single_path, 'meta.yaml')):
+                yield pth, single_path
+            else:
+                if not _semver:
+                    raise RepositoryException("You need semver support "
+                                              "for complex version matching")
+                if not os.path.isdir(single_path):
+                    continue
+                for single in os.listdir(single_path):
+                    try:
+                        semver.parse(single)
+                    except ValueError:
+                        fp = os.path.join(single_path, single)
+                        raise RepositoryException("Invalid repository"
+                                                  "content: %r" % fp)
+                    else:
+                        fp = os.path.join(single_path, single)
+                        if os.path.exists(os.path.join(fp, 'meta.yaml')):
+                            yield pth, fp
+
+    @classmethod
+    def repo_path(cls, repo_name):
+        return os.path.join(cls._REPOS_LOCATION, repo_name)
+
+    def create(self, source, link_only=False):
+        if not link_only:
+            try:
+                os.mkdir(self.fpath)
+            except OSError as e:
+                if e.errno == errno.EEXIST:
+                    raise RepositoryExists("Repository %s exists" % self.name)
+                else:
+                    raise
+            self._add_contents(source)
+        else:
+            os.symlink(source, self.fpath)
+
+    def update(self, source, overwrite=False):
+        self._add_contents(source, overwrite)
+
+    def _add_contents(self, source, overwrite=False):
+        cnts = self._list_source_contents(source)
+        for single_name, single_path in cnts:
+            self.add_single(single_name, single_path, overwrite)
+
+    def add_single(self, name, source, overwrite=False):
+        try:
+            metadata = read_meta(source)
+        except IOError as e:
+            if e.errno == errno.ENOENT:
+                raise RepositoryException(
+                    "meta.yaml not found: %s" % e.filename)
+            raise
+        version = metadata['version']
+        # TODO: (jnowak) sanitize version
+        target_path = os.path.join(self.fpath, name, version)
+        try:
+            shutil.copytree(source, target_path, symlinks=True)
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+            if not overwrite:
+                raise
+            shutil.rmtree(target_path)
+            shutil.copytree(source, target_path, symlinks=True)
+
+    def remove(self):
+        shutil.rmtree(self.fpath)
+
+    def remove_single(self, spec):
+        spec = self._parse_spec(spec)
+        if spec['version_sign'] != '==':
+            raise RepositoryException("Removal possible only with `==` sign")
+        path = self._make_version_path(spec)
+        shutil.rmtree(path)
+        return True
+
+    def iter_contents(self, resource_name=None):
+
+        def _single(single_path):
+            try:
+                for version in os.listdir(os.path.join(self.fpath,
+                                                       single_path)):
+                    yield {"name": single_path,
+                           'version': version}
+            except OSError:
+                return
+
+        if resource_name is None:
+            for single in os.listdir(self.fpath):
+                for gen in _single(single):
+                    yield gen
+        else:
+            for gen in _single(resource_name):
+                yield gen
+
+    def get_contents(self, resource_name=None):
+        out = defaultdict(list)
+        cnt = self.iter_contents(resource_name)
+        for curr in cnt:
+            out[curr['name']].append(curr['version'])
+        return out
+
+    @classmethod
+    def _parse_spec(cls, spec):
+        if isinstance(spec, dict):
+            return spec
+        if ':' in spec:
+            repos, version = spec.split(':', 1)
+        else:
+            repos = spec
+            version = None
+        if '/' in repos:
+            repo_name, resource_name = repos.split('/', 1)
+        else:
+            repo_name = 'resources'
+            resource_name = repos
+        if version is None:
+            version_sign = ">="
+        elif '>=' in version or '<=' in version or '==' in version:
+            version_sign = version[:2]
+            version = version[2:]
+        elif '>' in version or '<' in version:
+            version_sign = version[:1]
+            version = version[1:]
+        else:
+            version_sign = '=='
+        return {'repo': repo_name,
+                'resource_name': resource_name,
+                'version': version,
+                'version_sign': version_sign}
+
+    def _get_version(self, spec):
+        spec = self._parse_spec(spec)
+        version = spec['version']
+        version_sign = spec['version_sign']
+        resource_name = spec['resource_name']
+        if version_sign == '==':
+            return os.path.join(self.fpath, spec['resource_name'], version)
+        if not _semver:
+            raise RepositoryException("You need semver support "
+                                      "for complex version matching")
+        found = self.iter_contents(resource_name)
+        if version is None:
+            sc = semver.compare
+            sorted_vers = sorted(found,
+                                 cmp=lambda a, b: sc(a['version'],
+                                                     b['version']),
+                                 reverse=True)
+            if not sorted_vers:
+                raise ResourceNotFound(spec)
+            version = sorted_vers[0]['version']
+        else:
+            version = '{}{}'.format(version_sign, version)
+            matched = filter(lambda x: semver.match(x['version'], version),
+                             found)
+            sorted_vers = sorted(matched,
+                                 cmp=lambda a, b: semver.compare(a['version'],
+                                                                 b['version']),
+                                 reverse=True)
+            version = next((x['version'] for x in sorted_vers
+                            if semver.match(x['version'], version)),
+                           None)
+        if version is None:
+            raise ResourceNotFound(spec)
+        return version
+
+    def _make_version_path(self, spec, version=None):
+        spec = self._parse_spec(spec)
+        if version is None:
+            version = self._get_version(spec)
+        return os.path.join(self.fpath, spec['resource_name'], version)
+
+    def read_meta(self, spec):
+        path = self.get_path(spec)
+        return read_meta(path)
+
+    def get_path(self, spec):
+        spec = self._parse_spec(spec)
+        return self._make_version_path(spec)
+
+    @classmethod
+    def get_metadata(cls, spec):
+        spec = cls._parse_spec(spec)
+        repo = Repository(spec['repo'])
+        return repo.read_meta(spec)
+
+    @classmethod
+    def contains(cls, spec):
+        repo, spec = cls.parse(spec)
+        try:
+            version = repo._get_version(spec)
+            path = repo._make_version_path(spec, version=version)
+        except ResourceNotFound:
+            return False
+        return os.path.exists(path)
+
+    @classmethod
+    def what_version(cls, spec):
+        repo, spec = cls.parse(spec)
+        try:
+            version = repo._get_version(spec)
+            path = repo._make_version_path(spec, version=version)
+        except ResourceNotFound:
+            return False
+        if not os.path.exists(path):
+            return False
+        return version
+
+    @classmethod
+    def list_repos(cls):
+        return filter(lambda x:
+                      os.path.isdir(os.path.join(cls._REPOS_LOCATION,
+                                                 x)),
+                      os.listdir(cls._REPOS_LOCATION))
+
+    @classmethod
+    def parse(cls, spec):
+        spec = cls._parse_spec(spec)
+        return Repository(spec['repo']), spec

solar/core/resource/resource.py

@@ -25,6 +25,8 @@ from multipledispatch import dispatch
 import networkx
 
 
+from solar.core.resource.repository import read_meta
+from solar.core.resource.repository import Repository
 from solar.core.signals import get_mapping
 from solar.core.tags_set_parser import Expression
 from solar.core.tags_set_parser import get_string_tokens
@@ -36,19 +38,6 @@ from solar.events import api
 from solar import utils
 
 
-def read_meta(base_path):
-    base_meta_file = os.path.join(base_path, 'meta.yaml')
-
-    metadata = utils.yaml_load(base_meta_file)
-    metadata['version'] = '1.0.0'
-    metadata['base_path'] = os.path.abspath(base_path)
-    actions_path = os.path.join(metadata['base_path'], 'actions')
-    metadata['actions_path'] = actions_path
-    metadata['base_name'] = os.path.split(metadata['base_path'])[-1]
-
-    return metadata
-
-
 RESOURCE_STATE = Enum(
     'ResourceState', 'created operational removed error updated')
 
@@ -58,22 +47,28 @@ class Resource(object):
 
     # Create
     @dispatch(basestring, basestring)
-    def __init__(self, name, base_path, args=None, tags=None,
+    def __init__(self, name, spec, args=None, tags=None,
                  virtual_resource=None):
         args = args or {}
         self.name = name
-        if base_path:
+        if spec:
-            metadata = read_meta(base_path)
+            if spec.startswith('/'):
+                # it's full path, don't use repo
+                self.base_path = spec
+                metadata = read_meta(spec)
+            else:
+                repo, spec = Repository.parse(spec)
+                metadata = repo.get_metadata(spec)
+                self.base_path = repo.get_path(spec)
         else:
             metadata = deepcopy(self._metadata)
-
+            self.base_path = spec  # TODO: remove this old method?
-        self.base_path = base_path
 
         if tags is None:
             tags = []
         m_tags = metadata.get('tags', [])
         tags.extend(m_tags)
-        tags.append('resource={}'.format(metadata['id']))
+        tags.append('resource={}'.format(name))
 
         self.virtual_resource = virtual_resource
 

solar/core/resource/virtual_resource.py

@@ -32,23 +32,18 @@ from solar.events.controls import Dep
 from solar.events.controls import React
 
 
-def create(name, base_path, args=None, tags=None, virtual_resource=None):
+def create(name, spec, args=None, tags=None, virtual_resource=None):
     args = args or {}
-    if isinstance(base_path, provider.BaseProvider):
+    if isinstance(spec, provider.BaseProvider):
-        base_path = base_path.directory
+        spec = spec.directory
 
-    if not os.path.exists(base_path):
+    if is_virtual(spec):
-        raise Exception(
+        template = _compile_file(name, spec, args)
-            'Base resource does not exist: {0}'.format(base_path)
-        )
-
-    if is_virtual(base_path):
-        template = _compile_file(name, base_path, args)
         yaml_template = yaml.load(StringIO(template))
         rs = create_virtual_resource(name, yaml_template, tags)
     else:
         r = create_resource(name,
-                            base_path,
+                            spec,
                             args=args,
                             tags=tags,
                             virtual_resource=virtual_resource)
@@ -57,11 +52,11 @@ def create(name, base_path, args=None, tags=None, virtual_resource=None):
     return rs
 
 
-def create_resource(name, base_path, args=None, tags=None,
+def create_resource(name, spec, args=None, tags=None,
                     virtual_resource=None):
     args = args or {}
-    if isinstance(base_path, provider.BaseProvider):
+    if isinstance(spec, provider.BaseProvider):
-        base_path = base_path.directory
+        spec = spec.directory
 
     # filter connections from lists and dicts
     # will be added later
@@ -75,7 +70,7 @@ def create_resource(name, base_path, args=None, tags=None,
             return value
 
     args = {key: _filter(value) for key, value in args.items()}
-    r = Resource(name, base_path, args=args,
+    r = Resource(name, spec, args=args,
                  tags=tags, virtual_resource=virtual_resource)
     return r
 
@@ -130,18 +125,16 @@ def is_virtual(path):
 
 def create_resources(resources, tags=None):
     created_resources = []
-    cwd = os.getcwd()
     for r in resources:
         resource_name = r['id']
         args = r.get('values', {})
         node = r.get('location', None)
         values_from = r.get('values_from')
-        from_path = r.get('from', None)
+        spec = r.get('from', None)
         tags = r.get('tags', [])
-        base_path = os.path.join(cwd, from_path)
+        new_resources = create(resource_name, spec, args=args, tags=tags)
-        new_resources = create(resource_name, base_path, args=args, tags=tags)
         created_resources += new_resources
-        if not is_virtual(base_path):
+        if not is_virtual(spec):
             if node:
                 node = load_resource(node)
                 r = new_resources[0]

solar/test/conftest.py

@@ -16,6 +16,7 @@ import time
 
 import pytest
 
+from solar.core.resource.repository import Repository
 from solar.core.resource import Resource
 from solar.dblayer.model import get_bucket
 from solar.dblayer.model import Model
@@ -51,6 +52,14 @@ def setup(request):
         model.bucket = get_bucket(None, model, ModelMeta)
 
 
+@pytest.fixture(scope='session', autouse=True)
+def repos_path(tmpdir_factory):
+    Repository._REPOS_LOCATION = str(tmpdir_factory.mktemp('repositories'))
+    path = Repository._REPOS_LOCATION
+    repo = Repository('resources')
+    repo.create(path)
+
+
 def pytest_runtest_teardown(item, nextitem):
     ModelMeta.session_end(result=True)
     return nextitem

solar/test/test_resource_repository.py

@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import pytest
+import shutil
+from solar.core.resource.repository import Repository
+
+
+Repository._REPOS_LOCATION = '/tmp'
+
+
+_META_CONTENT = """
+handler: null
+version: {0}
+input:
+  a:
+    value: 1
+    schema: int!
+  name:
+    value: {1}
+  version:
+    value: {0}
+"""
+
+_VERSIONS = ('0.0.1', '0.0.2', '1.0.0', '1.4.7', '2.0.0')
+
+
+def generate_structure(target, versions='1.0.0'):
+    if isinstance(versions, basestring):
+        versions = (versions)
+    elif isinstance(versions, int):
+        versions = _VERSIONS[:versions]
+
+    for name in ('first', 'second', 'third'):
+        for version in versions:
+            cnt = _META_CONTENT.format(version, name)
+            fp = os.path.join(target, name, version)
+            os.makedirs(fp)
+            with open(os.path.join(fp, 'meta.yaml'), 'wb') as f:
+                f.write(cnt)
+
+
+def generator(request, tmpdir_factory):
+    try:
+        name = request.function.__name__
+    except AttributeError:
+        # function not available in module-scoped context
+        name = "module"
+    rp = str(tmpdir_factory.mktemp('{}-resources'.format(name)))
+    generate_structure(rp, 3)
+    return rp
+
+
+@pytest.fixture(scope='module', autouse=True)
+def repos_path(tmpdir_factory):
+    Repository._REPOS_LOCATION = str(tmpdir_factory.mktemp('repositories'))
+    return Repository._REPOS_LOCATION
+
+
+@pytest.fixture(scope="function")
+def ct(request, tmpdir_factory):
+    p = generator(request, tmpdir_factory)
+    request.addfinalizer(lambda: shutil.rmtree(p))
+    return p
+
+
+@pytest.fixture(scope="module")
+def repo_r(request, tmpdir_factory):
+    path = generator(request, tmpdir_factory)
+    r = Repository('rtest')
+    r.create(path)
+    return r
+
+
+@pytest.fixture(scope='function')
+def repo_w(request, tmpdir_factory):
+    path = generator(request, tmpdir_factory)
+    r = Repository('rwtest')
+    r.create(path)
+    request.addfinalizer(lambda: shutil.rmtree(path))
+    request.addfinalizer(lambda: r.remove())
+    return r
+
+
+def test_simple_create(ct):
+    r = Repository('test')
+    r.create(ct)
+    for k, v in r.get_contents().items():
+        assert len(v) == 3
+
+
+@pytest.mark.parametrize('spec, exp',
+                         (('rtest/first:0.0.1', True),
+                          ('rtest/first:0.0.5', False),
+                          ('invalid/first:0.0.5', False),
+                          ('invalid/first:0.0.1', False)))
+def test_simple_select(repo_r, spec, exp):
+    spec = Repository._parse_spec(spec)
+    assert Repository.contains(spec) is exp
+    if exp:
+        metadata = Repository.get_metadata(spec)
+        assert metadata['version'] == spec['version']
+        assert spec['version_sign'] == '=='
+
+
+@pytest.mark.parametrize('spec, exp',
+                         (('rtest/first', True),
+                          ('invalid/first', False)))
+def test_get_latest(repo_r, spec, exp):
+    spec = Repository._parse_spec(spec)
+    assert spec['version'] is None
+    assert Repository.contains(spec) is exp
+    if exp:
+        Repository.get_metadata(spec)
+        assert spec['version_sign'] == '>='
+
+
+@pytest.mark.parametrize('spec, exp, exp_ver',
+                         (('rtest/first:0.0.1', True, '0.0.1'),
+                          ('rtest/first:==0.0.1', True, '0.0.1'),
+                          ('rtest/first:==0.0.1', True, '0.0.1'),
+                          ('rtest/first:<=0.0.5', True, '0.0.2'),
+                          ('rtest/first:>=0.0.5', True, '1.0.0'),
+                          ('rtest/first:>=1.0.0', True, '1.0.0')))
+def test_guess_version_sharp(repo_r, spec, exp, exp_ver):
+    assert Repository.contains(spec) is exp
+    if exp:
+        metadata = Repository.get_metadata(spec)
+        assert metadata['version'] == exp_ver
+
+
+@pytest.mark.parametrize('spec, exp, exp_ver',
+                         (('rtest/first:<0.0.1', False, ''),
+                          ('rtest/first:<0.0.2', True, '0.0.1'),
+                          ('rtest/first:<0.0.5', True, '0.0.2'),
+                          ('rtest/first:>0.0.5', True, '1.0.0'),
+                          ('rtest/first:>1.0.0', False, '')))
+def test_guess_version_soft(repo_r, spec, exp, exp_ver):
+    assert Repository.contains(spec) is exp
+    if exp:
+        metadata = Repository.get_metadata(spec)
+        assert metadata['version'] == exp_ver
+
+
+@pytest.mark.parametrize('spec', ('rwtest/first:0.0.1',
+                                  'rwtest/first:==0.0.1'))
+def test_remove_single(repo_w, spec):
+    assert Repository.contains(spec)
+    repo_w.remove_single(spec)
+    assert Repository.contains(spec) is False
+
+
+def test_two_repos(tmpdir):
+    rp1 = str(tmpdir) + '/r1'
+    rp2 = str(tmpdir) + '/r2'
+    generate_structure(rp1, 2)
+    generate_structure(rp2, 5)
+    r1 = Repository('repo1')
+    r1.create(rp1)
+    r2 = Repository('repo2')
+    r2.create(rp2)
+    exp = set(['repo1', 'repo2'])
+    got = set(Repository.list_repos())
+    assert got.intersection(exp) == exp
+    assert Repository.contains('repo1/first:0.0.1')
+    assert Repository.contains('repo2/first:0.0.1')
+    assert Repository.contains('repo1/first:2.0.0') is False
+    assert Repository.contains('repo2/first:2.0.0')
+
+    r2.remove()
+    exp = set(['repo1'])
+    got = set(Repository.list_repos())
+    assert got.intersection(exp) == exp
+    assert Repository.contains('repo2/first:2.0.0') is False
+
+
+def test_update(repo_w, tmpdir):
+    rp = str(tmpdir) + '/second'
+    generate_structure(rp, 2)
+    with pytest.raises(OSError):
+        repo_w.update(rp)
+    repo_w.update(rp, overwrite=True)

solar/test/test_system_log_api.py

@@ -15,6 +15,7 @@
 import mock
 from pytest import mark
 
+from solar.core.resource import repository
 from solar.core.resource import resource
 from solar.core.resource import RESOURCE_STATE
 from solar.core import signals
@@ -149,12 +150,15 @@ def test_revert_removal():
     assert DBResource._c.obj_cache == {}
     # assert DBResource.bucket.get('test1').siblings == []
 
-    with mock.patch.object(resource, 'read_meta') as mread:
+    with mock.patch.object(repository.Repository, 'read_meta') as mread:
         mread.return_value = {
             'input': {'a': {'schema': 'str!'}},
             'id': 'mocked'
         }
-        change.revert(changes[0].uid)
+        with mock.patch.object(repository.Repository, 'get_path') as mpath:
+            mpath.return_value = 'x'
+
+            change.revert(changes[0].uid)
     ModelMeta.save_all_lazy()
     # assert len(DBResource.bucket.get('test1').siblings) == 1
 
@@ -194,7 +198,7 @@ def test_revert_removed_child():
     logitem = next(staged_log.collection())
     operations.move_to_commited(logitem.log_action)
 
-    with mock.patch.object(resource, 'read_meta') as mread:
+    with mock.patch.object(repository, 'read_meta') as mread:
         mread.return_value = {'input': {'a': {'schema': 'str!'}}}
         change.revert(logitem.uid)
 

solar/test/test_virtual_resource.py

@@ -52,8 +52,7 @@ def bad_event_type():
 def test_create_path_does_not_exists():
     with pytest.raises(Exception) as excinfo:
         vr.create('node1', '/path/does/not/exists')
-    err = 'Base resource does not exist: /path/does/not/exists'
+        assert excinfo.filename == '/path/does/not/exists'
-    assert str(excinfo.value) == err
 
 
 def test_create_resource():

test-requirements.txt

@@ -13,3 +13,7 @@ os-testr
 
 # to test if everything works on gevent
 gevent
+
+
+# semver for version tests
+semver