...
 
Commits (2)
......@@ -33,7 +33,8 @@ class DeploymentClient(AdministrationClient):
def __post_init__(self):
super().__post_init__()
config.load_kube_config(config_file=".kubeconfig")
config.load_kube_config(config_file="/kubectl_files/kubeconfig.yaml")
config.verify_ssl = False
self.appsV1client = k8sclient.AppsV1Api()
self.coreV1client = k8sclient.CoreV1Api()
self.netV1client = k8sclient.NetworkingV1beta1Api()
......@@ -66,11 +67,21 @@ class MarketDeploymentClient(DeploymentClient):
"deploy_access_manager",
"create_access_manager_service",
"create_access_manager_ingress_rule",
"migrate_access_manager",
"deploy_memo",
"create_memo_service",
"create_memo_ingress_rule"
]
def delete_job(self, name):
self.batchV1client.delete_namespaced_job(namespace=self.namespace, name=name)
for pod in self.coreV1client.list_namespaced_pod(namespace=self.namespace).items:
if name in pod.metadata.name:
self.coreV1client.delete_namespaced_pod(namespace=self.namespace, name=pod.metadata.name)
break
else:
continue
log.info("Deleted job {}".format(name))
def get_memo_url(self):
return get_memo_service_url(self.namespace)
......@@ -194,6 +205,7 @@ class MarketDeploymentClient(DeploymentClient):
log.info("Waiting for MEMO DB init to complete. Current status is: succeeded = {}".format(job.status.succeeded))
time.sleep(1)
if all_ok:
self.delete_job(name="memo-init-db")
break
def __step_deploy_memo(self):
......@@ -278,6 +290,26 @@ class MarketDeploymentClient(DeploymentClient):
body=ingress.get_object()
)
def __step_create_memo_ingress_rule(self):
ingress = Ingress(
namespace=self.namespace,
metadata_name="memo-ingress",
yaml_path="k8s/yaml/marketplace/ingress/memo.yaml",
)
ingress.insert_attribute(
"spec.rules.0.http.paths.0.path",
f"/catalyst/marketplace/{self.namespace}/memo(/|$)(.*)",
)
ingress.insert_attribute(
"metadata_annotations_nginx.ingress.kubernetes.io/rewrite-target",
"/memo/rest/$2",
sep="_"
)
self.netV1client.create_namespaced_ingress(
namespace=self.namespace,
body=ingress.get_object()
)
def __step_migrate_access_manager(self):
job = Job(
namespace=self.namespace,
......@@ -326,6 +358,16 @@ class VariantDeploymentClient(DeploymentClient):
"get_ib_service_url"
]
def delete_job(self, name):
self.batchV1client.delete_namespaced_job(namespace=self.namespace, name=name)
for pod in self.coreV1client.list_namespaced_pod(namespace=self.namespace).items:
if name in pod.metadata.name:
self.coreV1client.delete_namespaced_pod(namespace=self.namespace, name=pod.metadata.name)
break
else:
continue
log.info("Deleted job {}".format(name))
def deploy(self):
log.info(f"{self.namespace} - Starting variant deployment")
super().deploy()
......@@ -675,6 +717,21 @@ class VariantDeploymentClient(DeploymentClient):
namespace=self.namespace,
body=job.get_object()
)
while True:
done = False
for job in self.batchV1client.list_namespaced_job(namespace=self.namespace).items:
if job.metadata.name != f"information-broker-{variant_name}-migrations":
continue
if job.status.succeeded == 1:
log.info("Migrations finished.")
done = True
break
else:
log.info("Waiting for migrations to finish..")
time.sleep(1)
if done:
self.delete_job(name=f"information-broker-{variant_name}-migrations")
break
def __step_create_ib_jobs_create_super_user(self):
variant_name = self.variant_deployment_info.variant_name
......@@ -688,6 +745,21 @@ class VariantDeploymentClient(DeploymentClient):
namespace=self.namespace,
body=job.get_object()
)
while True:
done = False
for job in self.batchV1client.list_namespaced_job(namespace=self.namespace).items:
if job.metadata.name != f"information-broker-{variant_name}-create-su":
continue
if job.status.succeeded == 1:
log.info("SU created.")
done = True
break
else:
log.info("Waiting for SU create to finish..")
time.sleep(1)
if done:
self.delete_job(name=f"information-broker-{variant_name}-create-su")
break
def __step_create_ib_jobs_init_db(self):
variant_name = self.variant_deployment_info.variant_name
......@@ -701,6 +773,21 @@ class VariantDeploymentClient(DeploymentClient):
namespace=self.namespace,
body=job.get_object()
)
while True:
done = False
for job in self.batchV1client.list_namespaced_job(namespace=self.namespace).items:
if job.metadata.name != f"information-broker-{variant_name}-init-db":
continue
if job.status.succeeded == 1:
log.info("Init DB of IB finished.")
done = True
break
else:
log.info("Waiting for IB init DB to finish..")
time.sleep(1)
if done:
self.delete_job(name=f"information-broker-{variant_name}-init-db")
break
@dataclass
class VariantDeploymentDeletionClient(DeploymentClient):
......@@ -723,8 +810,7 @@ class MarketplaceDeploymentDeletionClient(DeploymentClient):
raise_exc = False
steps = [
"delete_memo_database_pv",
"delete_namespace",
"remove_maas_deployment"
"delete_namespace"
]
def __step_delete_memo_database_pv(self):
......
......@@ -18,7 +18,7 @@ variant_types = {
variant_forms = {
"electricity": "electric_energy",
"heat": "electric_energy",
"heat": "thermal_energy",
"flexibility": "ancillary_services",
"it-load": "it_load"
}
......
......@@ -133,6 +133,28 @@ class MemoUpdateClient(MessageClient):
return False
return res
def get_mcm_rel_url(self, variant):
if variant == 'electricity':
return 'mcm_el'
elif variant == 'flexibility':
return 'mcm_fl'
elif variant == 'heat':
return 'mcm_th'
elif variant == 'it-load':
return 'mcm_it'
return None
def get_mbm_rel_url(self, variant):
if variant == 'electricity':
return 'mbm_el'
elif variant == 'flexibility':
return 'mbm_fl'
elif variant == 'heat':
return 'mbm_th'
elif variant == 'it-load':
return 'mbm_it'
return None
def __format_message(self):
return json.dumps({
"type": self.type_,
......@@ -148,12 +170,12 @@ class MemoUpdateClient(MessageClient):
"password": self.ib_credentials['password']
},
"mcm": {
"url": "http://market-clearing-manager-{}-service:8080".format(self.variant),
"url": "http://market-clearing-manager-{}-service:8080/{}/rest".format(self.variant, self.get_mcm_rel_url(self.variant)),
"username": self.mcm_credentials['username'],
"password": self.mcm_credentials['password']
},
"mbm": {
"url": "http://market-billing-manager-{}-service:8080".format(self.variant),
"url": "http://market-billing-manager-{}-service:8080/{}/rest".format(self.variant, self.get_mbm_rel_url(self.variant)),
"username": self.mbm_credentials['username'],
"password": self.mbm_credentials['password']
}
......
......@@ -92,7 +92,7 @@ class NewVariantToExistingMarketplace(AdministrationClient):
locality_identifier=format_locality_identifier(self.region),
public_k8s_ip=settings.DEPLOYMENT_DEFAULTS['EXTERNAL_URL'],
url=get_memo_service_url(format_namespace(self.region)) + "/memo/rest/marketplaces/",
max_retries=200,
max_retries=10,
retry_interval=1,
)
# Validate that IB is ok
......
apiVersion: v1
kind: ConfigMap
metadata:
name: kubeconfig
data:
kubeconfig.yaml: |
apiVersion: v1
kind: Config
users:
- name: svcs-acct-dply
user:
token: <FILL_ME_IN>
clusters:
- cluster:
server: <K8S_API_URL>
insecure-skip-tls-verify: true
name: microk8s-cluster
contexts:
- context:
cluster: microk8s-cluster
user: svcs-acct-dply
name: svcs-acct-context
current-context: svcs-acct-context
......@@ -115,4 +115,11 @@ spec:
- name: RABBITMQ_HOST
value: maas-tasks-queue-service
volumeMounts:
- name: kubeconfig
mountPath: /kubectl_files
readOnly: true
volumes:
- name: kubeconfig
configMap:
name: kubeconfig
......@@ -116,3 +116,11 @@ spec:
- name: RABBITMQ_HOST
value: maas-tasks-queue-service
volumeMounts:
- name: kubeconfig
mountPath: /kubectl_files
readOnly: true
volumes:
- name: kubeconfig
configMap:
name: kubeconfig
\ No newline at end of file
......@@ -18,7 +18,6 @@ spec:
- name: access-manager
image: h2020catalyst/marketplace-access-manager:latest
imagePullPolicy: Always
command: ['pipenv', 'run', 'gunicorn', 'AccessManager.wsgi:application']
ports:
- containerPort: 80
env:
......
......@@ -28,5 +28,5 @@ spec:
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z memo-database-service 5432; do echo waiting for MEMO DB to start; sleep 1; done"]
command: ['sh', '-c', "until nc -w 1 -z memo-database-service 5432; do echo waiting for MEMO DB to start; sleep 5; done"]
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: access-manager-ingress
namespace: namespace-placeholder
annotations:
nginx.ingress.kubernetes.io/rewrite-target: /memo/rest/$2
nginx.ingress.kubernetes.io/ssl-redirect: "false"
nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
spec:
rules:
- http:
paths:
- path: /catalyst/marketplace/namespace-placeholder/memo(/|$)(.*)
backend:
serviceName: memo-service
servicePort: 80
......@@ -54,5 +54,9 @@ spec:
secretKeyRef:
name: information-broker-electricity-credentials
key: email
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-electricity-database-service 5432; do echo waiting for information-broker-electricity-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,9 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-electricity-database-service 5432; do echo waiting for information-broker-electricity-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,9 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-electricity-database-service 5432; do echo waiting for information-broker-electricity-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -54,5 +54,9 @@ spec:
secretKeyRef:
name: information-broker-flexibility-credentials
key: email
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-flexibility-database-service 5432; do echo waiting for information-broker-flexibility-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,9 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-flexibility-database-service 5432; do echo waiting for information-broker-flexibility-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,10 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-flexibility-database-service 5432; do echo waiting for information-broker-flexibility-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -54,5 +54,10 @@ spec:
secretKeyRef:
name: information-broker-heat-credentials
key: email
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-heat-database-service 5432; do echo waiting for information-broker-heat-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,10 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-heat-database-service 5432; do echo waiting for information-broker-heat-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,10 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-heat-database-service 5432; do echo waiting for information-broker-heat-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -54,5 +54,10 @@ spec:
secretKeyRef:
name: information-broker-it-load-credentials
key: email
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-it-load-database-service 5432; do echo waiting for information-broker-it-load-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,10 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-it-load-database-service 5432; do echo waiting for information-broker-it-load-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -36,6 +36,10 @@ spec:
- name: DJANGO_SETTINGS_MODULE
value: information_broker.settings.production
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z information-broker-it-load-database-service 5432; do echo waiting for information-broker-it-load-database-service to start; sleep 1; done"]
restartPolicy: Never
backoffLimit: 5
......@@ -21,6 +21,10 @@ spec:
volumeMounts:
- name: database-init
mountPath: /initdb/
initContainers:
- name: wait-for-db
image: busybox:latest
command: ['sh', '-c', "until nc -w 1 -z memo-database-service 5432; do echo waiting for MEMO DB to start; sleep 1; done"]
volumes:
- name: database-init
......