diff --git a/.gitmodules b/.gitmodules index 9c8e646440..602915f94f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "src/github.com/cloudfoundry/blobstore_url_signer"] - path = src/github.com/cloudfoundry/blobstore_url_signer - url = https://github.com/cloudfoundry/blobstore_url_signer [submodule "src/cloud_controller_ng"] path = src/cloud_controller_ng url = https://github.com/cloudfoundry/cloud_controller_ng diff --git a/README.md b/README.md index 6e449c23cd..d917fad16b 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,6 @@ This is the [bosh release](http://bosh.io/docs/release.html) for Cloud Foundry's * [Cloud Controller](https://github.com/cloudfoundry/cloud_controller_ng): The primary API of Cloud Foundry. * [Cloud Controller Clock](https://github.com/cloudfoundry/cloud_controller_ng): Triggers periodic jobs for the Cloud Controller. * [Cloud Controller Workers](https://github.com/cloudfoundry/cloud_controller_ng): Execute background jobs for the Cloud Controller. -* [Webdav Blobstore](https://github.com/cloudfoundry/capi-release/tree/develop/jobs/blobstore): An optional stand-alone blobstore for the Cloud Controller. * [NFS Mounter](https://github.com/cloudfoundry/capi-release/tree/develop/jobs/nfs_mounter): Connects Cloud Controller with an NFS blobstore. * [CC Uploader](https://github.com/cloudfoundry/cc-uploader): Uploads files from [Diego](https://github.com/cloudfoundry/diego-release) to the Cloud Controller. * [TPS Watcher](https://github.com/cloudfoundry/tps): Reports crash events from Diego to the Cloud Controller. diff --git a/jobs/blobstore/monit b/jobs/blobstore/monit index cd18d38a50..718605bdc6 100644 --- a/jobs/blobstore/monit +++ b/jobs/blobstore/monit @@ -5,12 +5,6 @@ check process blobstore_nginx stop program "/var/vcap/jobs/bpm/bin/bpm stop blobstore -p nginx" group vcap - check process blobstore_url_signer - with pidfile /var/vcap/sys/run/bpm/blobstore/url_signer.pid - start program "/var/vcap/jobs/bpm/bin/bpm start blobstore -p url_signer" - stop program "/var/vcap/jobs/bpm/bin/bpm stop blobstore -p url_signer" - group vcap - <% else %> # This file intentionally left blank # Blobstore job is only run on the bootstrap instance diff --git a/jobs/blobstore/spec b/jobs/blobstore/spec index 650093bfef..10ff0ad362 100644 --- a/jobs/blobstore/spec +++ b/jobs/blobstore/spec @@ -19,7 +19,6 @@ templates: packages: - nginx_webdav -- blobstore_url_signer - capi_utils consumes: @@ -60,6 +59,10 @@ properties: description: The TCP port on which the internal blobstore server listens default: 4443 + blobstore.internal_hostname: + description: Internal hostname used by the singleton blobstore + default: "blobstore.service.cf.internal" + blobstore.tls.cert: description: The PEM-encoded certificate (optionally as a certificate chain) for serving blobs over TLS/SSL diff --git a/jobs/blobstore/templates/blobstore.conf.erb b/jobs/blobstore/templates/blobstore.conf.erb index 6448eb15c2..46126fd42b 100644 --- a/jobs/blobstore/templates/blobstore.conf.erb +++ b/jobs/blobstore/templates/blobstore.conf.erb @@ -56,14 +56,10 @@ server { return 404; } -upstream blob_url_signer { - server unix:/var/vcap/data/blobstore/signer.sock; -} - # Internal server server { listen <%= p('blobstore.tls.port') %> ssl; - server_name blobstore.service.cf.internal; + server_name <%= p('blobstore.internal_hostname', 'blobstore.service.cf.internal') %>; ssl_certificate /var/vcap/jobs/blobstore/ssl/blobstore.crt; ssl_certificate_key /var/vcap/jobs/blobstore/ssl/blobstore.key; @@ -94,13 +90,6 @@ server { alias /var/vcap/store/shared/; } - location /sign { - auth_basic "Blobstore Signing"; - auth_basic_user_file write_users; - - proxy_pass http://blob_url_signer; - } - <%= signed_location %> # ensure the contents of this location block always match the public server /read/ location block diff --git a/jobs/blobstore/templates/bpm.yml.erb b/jobs/blobstore/templates/bpm.yml.erb index a17ca0537b..86233fdb8a 100644 --- a/jobs/blobstore/templates/bpm.yml.erb +++ b/jobs/blobstore/templates/bpm.yml.erb @@ -7,10 +7,3 @@ processes: writable: true ephemeral_disk: true persistent_disk: true -- name: url_signer - executable: /var/vcap/packages/blobstore_url_signer/bin/blobstore_url_signer - ephemeral_disk: true - args: - - --secret=<%= p("blobstore.secure_link.secret") %> - - --network=unix - - --laddr=/var/vcap/data/blobstore/signer.sock diff --git a/jobs/blobstore_benchmark/templates/cloud_controller_ng.yml.erb b/jobs/blobstore_benchmark/templates/cloud_controller_ng.yml.erb index 3beb3c2ea9..c1f750727f 100644 --- a/jobs/blobstore_benchmark/templates/cloud_controller_ng.yml.erb +++ b/jobs/blobstore_benchmark/templates/cloud_controller_ng.yml.erb @@ -60,7 +60,7 @@ section = cc_cfg[k] next unless section.is_a?(Hash) - %w[connection_config webdav_config].each do |hk| + %w[connection_config].each do |hk| section[hk] = {} if section.key?(hk) && section[hk].nil? end diff --git a/jobs/cc_deployment_updater/spec b/jobs/cc_deployment_updater/spec index ae4c3152f3..a004f89164 100644 --- a/jobs/cc_deployment_updater/spec +++ b/jobs/cc_deployment_updater/spec @@ -23,10 +23,6 @@ templates: mutual_tls.crt.erb: config/certs/mutual_tls.crt mutual_tls.key.erb: config/certs/mutual_tls.key credhub_ca.crt.erb: config/certs/credhub_ca.crt - packages_ca_cert.pem.erb: config/certs/packages_ca_cert.pem - resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem - droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem - buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem prom_scraper_config.yml.erb: config/prom_scraper_config.yml scrape.crt.erb: config/certs/scrape.crt scrape.key.erb: config/certs/scrape.key diff --git a/jobs/cc_deployment_updater/templates/buildpacks_ca_cert.pem.erb b/jobs/cc_deployment_updater/templates/buildpacks_ca_cert.pem.erb deleted file mode 100644 index e8d78ef934..0000000000 --- a/jobs/cc_deployment_updater/templates/buildpacks_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= link("cloud_controller_internal").p('cc.buildpacks.webdav_config.ca_cert') %> diff --git a/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb b/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb index b8211663b5..a5af130b13 100644 --- a/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb +++ b/jobs/cc_deployment_updater/templates/cloud_controller_ng.yml.erb @@ -200,15 +200,6 @@ storage_cli_optional_flags: <%= link("cloud_controller_internal").p("cc.storage_ resource_pool: blobstore_type: <%= link("cloud_controller_internal").p("cc.resource_pool.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= link("cloud_controller_internal").p("cc.resource_pool.webdav_config.blobstore_timeout") %> - public_endpoint: <%= link("cloud_controller_internal").p("cc.resource_pool.webdav_config.public_endpoint") %> - private_endpoint: <%= link("cloud_controller_internal").p("cc.resource_pool.webdav_config.private_endpoint") %> - username: <%= link("cloud_controller_internal").p("cc.resource_pool.webdav_config.username") %> - password: <%= yaml_escape(link("cloud_controller_internal").p("cc.resource_pool.webdav_config.password")) %> - <% if link("cloud_controller_internal").p("cc.resource_pool.webdav_config.ca_cert") != "" && link("cloud_controller_internal").p("cc.resource_pool.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/resource_pool_ca_cert.pem" - <% end %> minimum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.minimum_size") %> maximum_size: <%= link("cloud_controller_internal").p("cc.resource_pool.maximum_size") %> resource_directory_key: <%= link("cloud_controller_internal").p("cc.resource_pool.resource_directory_key") %> @@ -219,15 +210,6 @@ resource_pool: packages: blobstore_type: <%= link("cloud_controller_internal").p("cc.packages.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= link("cloud_controller_internal").p("cc.packages.webdav_config.blobstore_timeout") %> - public_endpoint: <%= link("cloud_controller_internal").p("cc.packages.webdav_config.public_endpoint") %> - private_endpoint: <%= link("cloud_controller_internal").p("cc.packages.webdav_config.private_endpoint") %> - username: <%= link("cloud_controller_internal").p("cc.packages.webdav_config.username") %> - password: <%= yaml_escape(link("cloud_controller_internal").p("cc.packages.webdav_config.password")) %> - <% if link("cloud_controller_internal").p("cc.packages.webdav_config.ca_cert") != "" && link("cloud_controller_internal").p("cc.packages.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/packages_ca_cert.pem" - <% end %> app_package_directory_key: <%= link("cloud_controller_internal").p("cc.packages.app_package_directory_key") %> <% if (pk_provider = link("cloud_controller_internal").p("cc.packages.blobstore_provider", nil)) %> blobstore_provider: <%= pk_provider %> @@ -238,15 +220,6 @@ packages: droplets: blobstore_type: <%= link("cloud_controller_internal").p("cc.droplets.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= link("cloud_controller_internal").p("cc.droplets.webdav_config.blobstore_timeout") %> - public_endpoint: <%= link("cloud_controller_internal").p("cc.droplets.webdav_config.public_endpoint") %> - private_endpoint: <%= link("cloud_controller_internal").p("cc.droplets.webdav_config.private_endpoint") %> - username: <%= link("cloud_controller_internal").p("cc.droplets.webdav_config.username") %> - password: <%= yaml_escape(link("cloud_controller_internal").p("cc.droplets.webdav_config.password")) %> - <% if link("cloud_controller_internal").p("cc.droplets.webdav_config.ca_cert") != "" && link("cloud_controller_internal").p("cc.droplets.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/droplets_ca_cert.pem" - <% end %> droplet_directory_key: <%= link("cloud_controller_internal").p("cc.droplets.droplet_directory_key") %> <% if (dp_provider = link("cloud_controller_internal").p("cc.droplets.blobstore_provider", nil)) %> blobstore_provider: <%= dp_provider %> @@ -255,15 +228,6 @@ droplets: buildpacks: blobstore_type: <%= link("cloud_controller_internal").p("cc.buildpacks.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= link("cloud_controller_internal").p("cc.buildpacks.webdav_config.blobstore_timeout") %> - public_endpoint: <%= link("cloud_controller_internal").p("cc.buildpacks.webdav_config.public_endpoint") %> - private_endpoint: <%= link("cloud_controller_internal").p("cc.buildpacks.webdav_config.private_endpoint") %> - username: <%= link("cloud_controller_internal").p("cc.buildpacks.webdav_config.username") %> - password: <%= yaml_escape(link("cloud_controller_internal").p("cc.buildpacks.webdav_config.password")) %> - <% if link("cloud_controller_internal").p("cc.buildpacks.webdav_config.ca_cert") != "" && link("cloud_controller_internal").p("cc.buildpacks.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cc_deployment_updater/config/certs/buildpacks_ca_cert.pem" - <% end %> buildpack_directory_key: <%= link("cloud_controller_internal").p("cc.buildpacks.buildpack_directory_key") %> <% if (bp_provider = link("cloud_controller_internal").p("cc.buildpacks.blobstore_provider", nil)) %> blobstore_provider: <%= bp_provider %> diff --git a/jobs/cc_deployment_updater/templates/droplets_ca_cert.pem.erb b/jobs/cc_deployment_updater/templates/droplets_ca_cert.pem.erb deleted file mode 100644 index 1f8fccfe60..0000000000 --- a/jobs/cc_deployment_updater/templates/droplets_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= link("cloud_controller_internal").p('cc.droplets.webdav_config.ca_cert') %> diff --git a/jobs/cc_deployment_updater/templates/packages_ca_cert.pem.erb b/jobs/cc_deployment_updater/templates/packages_ca_cert.pem.erb deleted file mode 100644 index 56bd3fae7f..0000000000 --- a/jobs/cc_deployment_updater/templates/packages_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= link("cloud_controller_internal").p('cc.packages.webdav_config.ca_cert') %> diff --git a/jobs/cc_deployment_updater/templates/resource_pool_ca_cert.pem.erb b/jobs/cc_deployment_updater/templates/resource_pool_ca_cert.pem.erb deleted file mode 100644 index fdd3789c21..0000000000 --- a/jobs/cc_deployment_updater/templates/resource_pool_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= link("cloud_controller_internal").p('cc.resource_pool.webdav_config.ca_cert') %> diff --git a/jobs/cloud_controller_clock/spec b/jobs/cloud_controller_clock/spec index b5745d9c8e..232e6eb46e 100644 --- a/jobs/cloud_controller_clock/spec +++ b/jobs/cloud_controller_clock/spec @@ -210,26 +210,8 @@ properties: cc.resource_pool.connection_config: description: "Storage CLI connection configuration" cc.resource_pool.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.resource_pool.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.resource_pool.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.resource_pool.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.resource_pool.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.resource_pool.minimum_size: description: "Minimum size of a resource to add to the pool" default: 65536 @@ -246,26 +228,8 @@ properties: cc.packages.connection_config: description: "Storage CLI connection configuration" cc.packages.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.packages.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.packages.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.packages.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.packages.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.packages.app_package_directory_key: description: "Directory (bucket) used store app packages. It does not have be pre-created." default: "cc-packages" @@ -279,26 +243,8 @@ properties: cc.droplets.connection_config: description: "Storage CLI connection configuration" cc.droplets.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.droplets.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.droplets.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.droplets.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.droplets.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.droplets.droplet_directory_key: description: "Directory (bucket) used store droplets. It does not have be pre-created." default: "cc-droplets" @@ -309,26 +255,8 @@ properties: cc.buildpacks.connection_config: description: "Storage CLI connection configuration" cc.buildpacks.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.buildpacks.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.buildpacks.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.buildpacks.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.buildpacks.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.buildpacks.buildpack_directory_key: description: "Directory (bucket) used store buildpacks. It does not have be pre-created." default: "cc-buildpacks" diff --git a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb index 650ef6a0b4..2ac8cb066e 100644 --- a/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_clock/templates/cloud_controller_ng.yml.erb @@ -233,15 +233,6 @@ default_health_check_timeout: <%= p("cc.default_health_check_timeout") %> resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.resource_pool.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.resource_pool.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.resource_pool.webdav_config.private_endpoint") %> - username: <%= p("cc.resource_pool.webdav_config.username") %> - password: <%= yaml_escape(p("cc.resource_pool.webdav_config.password")) %> - <% if p("cc.resource_pool.webdav_config.ca_cert") != "" && p("cc.resource_pool.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_clock/config/certs/resource_pool_ca_cert.pem" - <% end %> minimum_size: <%= p("cc.resource_pool.minimum_size") %> maximum_size: <%= p("cc.resource_pool.maximum_size") %> resource_directory_key: <%= p("cc.resource_pool.resource_directory_key") %> @@ -249,42 +240,15 @@ resource_pool: packages: blobstore_type: <%= p("cc.packages.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.packages.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.packages.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.packages.webdav_config.private_endpoint") %> - username: <%= p("cc.packages.webdav_config.username") %> - password: <%= yaml_escape(p("cc.packages.webdav_config.password")) %> - <% if p("cc.packages.webdav_config.ca_cert") != "" && p("cc.packages.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_clock/config/certs/packages_ca_cert.pem" - <% end %> app_package_directory_key: <%= p("cc.packages.app_package_directory_key") %> max_package_size: <%= p("cc.packages.max_package_size") %> droplets: blobstore_type: <%= p("cc.droplets.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.droplets.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.droplets.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.droplets.webdav_config.private_endpoint") %> - username: <%= p("cc.droplets.webdav_config.username") %> - password: <%= yaml_escape(p("cc.droplets.webdav_config.password")) %> - <% if p("cc.droplets.webdav_config.ca_cert") != "" && p("cc.droplets.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_clock/config/certs/droplets_ca_cert.pem" - <% end %> droplet_directory_key: <%= p("cc.droplets.droplet_directory_key") %> buildpacks: blobstore_type: <%= p("cc.buildpacks.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.buildpacks.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.buildpacks.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.buildpacks.webdav_config.private_endpoint") %> - username: <%= p("cc.buildpacks.webdav_config.username") %> - password: <%= yaml_escape(p("cc.buildpacks.webdav_config.password")) %> - <% if p("cc.buildpacks.webdav_config.ca_cert") != "" && p("cc.buildpacks.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_clock/config/certs/buildpacks_ca_cert.pem" - <% end %> buildpack_directory_key: <%= p("cc.buildpacks.buildpack_directory_key") %> db_encryption_key: <%= p("cc.db_encryption_key") %> diff --git a/jobs/cloud_controller_ng/spec b/jobs/cloud_controller_ng/spec index ba7e1b0d25..3312c9ac54 100644 --- a/jobs/cloud_controller_ng/spec +++ b/jobs/cloud_controller_ng/spec @@ -4,6 +4,7 @@ name: cloud_controller_ng description: "The Cloud Controller provides primary Cloud Foundry API that is by the CF CLI. The Cloud Controller uses a database to keep tables for organizations, spaces, apps, services, service instances, user roles, and more. Typically multiple instances of Cloud Controller are load balanced." templates: + blobstore_waiter.sh.erb: bin/blobstore_waiter.sh bin/cloud_controller_ng.erb: bin/cloud_controller_ng bin/local_worker.erb: bin/local_worker bin/nginx_newrelic_plugin.erb: bin/nginx_newrelic_plugin @@ -15,13 +16,10 @@ templates: post-restore-unlock.sh.erb: bin/bbr/post-restore-unlock cloud_controller_ng.yml.erb: config/cloud_controller_ng.yml - blobstore_waiter.sh.erb: bin/blobstore_waiter.sh - buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem cloud_controller_api_health_check.erb: bin/cloud_controller_ng_health_check ccng_monit_http_healthcheck.sh.erb: bin/ccng_monit_http_healthcheck console.erb: bin/console drain.sh.erb: bin/drain - droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem logcache_tls.crt.erb: config/certs/logcache_tls.crt logcache_tls.key.erb: config/certs/logcache_tls.key logcache_tls_ca.crt.erb: config/certs/logcache_tls_ca.crt @@ -43,13 +41,11 @@ templates: newrelic_plugin.yml.erb: config/newrelic_plugin.yml nginx.conf.erb: config/nginx.conf nginx_maintenance.conf.erb: config/nginx_maintenance.conf - packages_ca_cert.pem.erb: config/certs/packages_ca_cert.pem perform_blobstore_benchmarks.erb: bin/perform_blobstore_benchmarks post-start.sh.erb: bin/post-start pre-start.sh.erb: bin/pre-start public_upload.conf.erb: config/public_upload.conf restart_drain.sh.erb: bin/restart_drain - resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain ruby_version.sh.erb: bin/ruby_version.sh seed_db.sh.erb: bin/seed_db @@ -121,12 +117,6 @@ provides: - cc.buildpacks.blobstore_provider - cc.buildpacks.blobstore_type - cc.buildpacks.buildpack_directory_key - - cc.buildpacks.webdav_config.blobstore_timeout - - cc.buildpacks.webdav_config.ca_cert - - cc.buildpacks.webdav_config.password - - cc.buildpacks.webdav_config.private_endpoint - - cc.buildpacks.webdav_config.public_endpoint - - cc.buildpacks.webdav_config.username - cc.cpu_weight_min_memory - cc.cpu_weight_max_memory - cc.credential_references.interpolate_service_bindings @@ -145,12 +135,6 @@ provides: - cc.droplets.blobstore_provider - cc.droplets.blobstore_type - cc.droplets.droplet_directory_key - - cc.droplets.webdav_config.blobstore_timeout - - cc.droplets.webdav_config.ca_cert - - cc.droplets.webdav_config.password - - cc.droplets.webdav_config.private_endpoint - - cc.droplets.webdav_config.public_endpoint - - cc.droplets.webdav_config.username - cc.external_host - cc.external_port - cc.external_protocol @@ -170,23 +154,11 @@ provides: - cc.packages.blobstore_type - cc.packages.max_package_size - cc.packages.max_valid_packages_stored - - cc.packages.webdav_config.blobstore_timeout - - cc.packages.webdav_config.ca_cert - - cc.packages.webdav_config.password - - cc.packages.webdav_config.private_endpoint - - cc.packages.webdav_config.public_endpoint - - cc.packages.webdav_config.username - cc.resource_pool.blobstore_provider - cc.resource_pool.blobstore_type - cc.resource_pool.maximum_size - cc.resource_pool.minimum_size - cc.resource_pool.resource_directory_key - - cc.resource_pool.webdav_config.blobstore_timeout - - cc.resource_pool.webdav_config.ca_cert - - cc.resource_pool.webdav_config.password - - cc.resource_pool.webdav_config.private_endpoint - - cc.resource_pool.webdav_config.public_endpoint - - cc.resource_pool.webdav_config.username - cc.stacks - cc.staging_timeout_in_seconds - cc.staging_upload_password @@ -247,6 +219,10 @@ properties: description: "specifies that the job is allowed to skip ssl cert verification" default: false + blobstore.internal_hostname: + description: "Internal hostname used by the singleton blobstore" + default: "blobstore.service.cf.internal" + system_domain: description: "Domain reserved for CF operator, base URL where the login, uaa, and other non-user apps listen" system_domain_organization: @@ -516,26 +492,8 @@ properties: cc.resource_pool.connection_config: description: "Storage CLI connection configuration" cc.resource_pool.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.resource_pool.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.resource_pool.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.resource_pool.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.resource_pool.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.resource_pool.minimum_size: description: "Minimum size of a resource to add to the pool" default: 65536 @@ -552,26 +510,8 @@ properties: cc.packages.connection_config: description: "Storage CLI connection configuration" cc.packages.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.packages.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.packages.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.packages.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.packages.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.packages.app_package_directory_key: description: "Directory (bucket) used store app packages. It does not have be pre-created. Should contain only alphanumeric characters, '-', '_', and '.'" default: "cc-packages" @@ -588,26 +528,8 @@ properties: cc.droplets.connection_config: description: "Storage CLI connection configuration" cc.droplets.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.droplets.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.droplets.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.droplets.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.droplets.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.droplets.droplet_directory_key: description: "Directory (bucket) used store droplets. It does not have be pre-created. Should contain only alphanumeric characters, '-', '_', and '.'" default: "cc-droplets" @@ -621,26 +543,8 @@ properties: cc.buildpacks.connection_config: description: "Storage CLI connection configuration" cc.buildpacks.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.buildpacks.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.buildpacks.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.buildpacks.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.buildpacks.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.ca_cert: - description: "The ca cert to use when communicating with webdav" - default: "" cc.buildpacks.buildpack_directory_key: description: "Directory (bucket) used store buildpacks. It does not have be pre-created. Should contain only alphanumeric characters, '-', '_', and '.'" default: "cc-buildpacks" diff --git a/jobs/cloud_controller_ng/templates/buildpacks_ca_cert.pem.erb b/jobs/cloud_controller_ng/templates/buildpacks_ca_cert.pem.erb deleted file mode 120000 index 9c9bc307ac..0000000000 --- a/jobs/cloud_controller_ng/templates/buildpacks_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/buildpacks_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb index c7f69f1ca7..e7ecda7862 100644 --- a/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_ng/templates/cloud_controller_ng.yml.erb @@ -344,59 +344,23 @@ default_quota_definition: <%= p("cc.default_quota_definition") %> resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.resource_pool.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.resource_pool.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.resource_pool.webdav_config.private_endpoint") %> - username: <%= p("cc.resource_pool.webdav_config.username") %> - password: <%= yaml_escape(p("cc.resource_pool.webdav_config.password")) %> - <% if p("cc.resource_pool.webdav_config.ca_cert") != "" && p("cc.resource_pool.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_ng/config/certs/resource_pool_ca_cert.pem" - <% end %> minimum_size: <%= p("cc.resource_pool.minimum_size") %> maximum_size: <%= p("cc.resource_pool.maximum_size") %> resource_directory_key: <%= p("cc.resource_pool.resource_directory_key") %> packages: blobstore_type: <%= p("cc.packages.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.packages.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.packages.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.packages.webdav_config.private_endpoint") %> - username: <%= p("cc.packages.webdav_config.username") %> - password: <%= yaml_escape(p("cc.packages.webdav_config.password")) %> - <% if p("cc.packages.webdav_config.ca_cert") != "" && p("cc.packages.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_ng/config/certs/packages_ca_cert.pem" - <% end %> app_package_directory_key: <%= p("cc.packages.app_package_directory_key") %> max_valid_packages_stored: <%= p("cc.packages.max_valid_packages_stored") %> max_package_size: <%= p("cc.packages.max_package_size") %> droplets: blobstore_type: <%= p("cc.droplets.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.droplets.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.droplets.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.droplets.webdav_config.private_endpoint") %> - username: <%= p("cc.droplets.webdav_config.username") %> - password: <%= yaml_escape(p("cc.droplets.webdav_config.password")) %> - <% if p("cc.droplets.webdav_config.ca_cert") != "" && p("cc.droplets.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_ng/config/certs/droplets_ca_cert.pem" - <% end %> droplet_directory_key: <%= p("cc.droplets.droplet_directory_key") %> max_staged_droplets_stored: <%= p("cc.droplets.max_staged_droplets_stored") %> buildpacks: blobstore_type: <%= p("cc.buildpacks.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.buildpacks.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.buildpacks.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.buildpacks.webdav_config.private_endpoint") %> - username: <%= p("cc.buildpacks.webdav_config.username") %> - password: <%= yaml_escape(p("cc.buildpacks.webdav_config.password")) %> - <% if p("cc.buildpacks.webdav_config.ca_cert") != "" && p("cc.buildpacks.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_ng/config/certs/buildpacks_ca_cert.pem" - <% end %> buildpack_directory_key: <%= p("cc.buildpacks.buildpack_directory_key") %> db_encryption_key: <%= p("cc.db_encryption_key") %> diff --git a/jobs/cloud_controller_ng/templates/droplets_ca_cert.pem.erb b/jobs/cloud_controller_ng/templates/droplets_ca_cert.pem.erb deleted file mode 120000 index d19041ffde..0000000000 --- a/jobs/cloud_controller_ng/templates/droplets_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/droplets_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/packages_ca_cert.pem.erb b/jobs/cloud_controller_ng/templates/packages_ca_cert.pem.erb deleted file mode 120000 index 1d24977063..0000000000 --- a/jobs/cloud_controller_ng/templates/packages_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/packages_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_ng/templates/resource_pool_ca_cert.pem.erb b/jobs/cloud_controller_ng/templates/resource_pool_ca_cert.pem.erb deleted file mode 120000 index 25569fb402..0000000000 --- a/jobs/cloud_controller_ng/templates/resource_pool_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/resource_pool_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_worker/spec b/jobs/cloud_controller_worker/spec index 855d6743db..447fab534e 100644 --- a/jobs/cloud_controller_worker/spec +++ b/jobs/cloud_controller_worker/spec @@ -4,6 +4,7 @@ name: cloud_controller_worker description: "Cloud Controller worker processes background tasks submitted via the." templates: + blobstore_waiter.sh.erb: bin/blobstore_waiter.sh bpm.yml.erb: config/bpm.yml bin/cloud_controller_worker.erb: bin/cloud_controller_worker pre-backup-lock.sh.erb: bin/bbr/pre-backup-lock @@ -17,13 +18,8 @@ templates: stacks.yml.erb: config/stacks.yml ruby_version.sh.erb: bin/ruby_version.sh console.erb: bin/console - blobstore_waiter.sh.erb: bin/blobstore_waiter.sh post-start.sh.erb: bin/post-start pre-start.sh.erb: bin/pre-start - buildpacks_ca_cert.pem.erb: config/certs/buildpacks_ca_cert.pem - droplets_ca_cert.pem.erb: config/certs/droplets_ca_cert.pem - packages_ca_cert.pem.erb: config/certs/packages_ca_cert.pem - resource_pool_ca_cert.pem.erb: config/certs/resource_pool_ca_cert.pem shutdown_drain.rb.erb: bin/shutdown_drain mutual_tls_ca.crt.erb: config/certs/mutual_tls_ca.crt mutual_tls.crt.erb: config/certs/mutual_tls.crt @@ -72,6 +68,10 @@ properties: description: "specifies that the job is allowed to skip ssl cert verification" default: false + blobstore.internal_hostname: + description: "Internal hostname used by the singleton blobstore" + default: "blobstore.service.cf.internal" + system_domain: description: "Domain reserved for CF operator, base URL where the login, uaa, and other non-user apps listen" @@ -151,26 +151,8 @@ properties: cc.resource_pool.connection_config: description: "Storage CLI connection configuration" cc.resource_pool.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.resource_pool.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.resource_pool.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.resource_pool.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.resource_pool.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.resource_pool.webdav_config.ca_cert: - description: "The CA certificate to use when communicating with webdav" - default: "" cc.resource_pool.minimum_size: description: "Minimum size of a resource to add to the pool" default: 65536 @@ -187,26 +169,8 @@ properties: cc.packages.connection_config: description: "Storage CLI connection configuration" cc.packages.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.packages.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.packages.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.packages.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.packages.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.packages.webdav_config.ca_cert: - description: "The CA certificate to use when communicating with webdav" - default: "" cc.packages.app_package_directory_key: description: "Directory (bucket) used store app packages. It does not have be pre-created." default: "cc-packages" @@ -220,26 +184,8 @@ properties: cc.droplets.connection_config: description: "Storage CLI connection configuration" cc.droplets.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.droplets.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.droplets.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.droplets.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.droplets.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.droplets.webdav_config.ca_cert: - description: "The CA certificate to use when communicating with webdav" - default: "" cc.droplets.droplet_directory_key: description: "Directory (bucket) used store droplets. It does not have be pre-created." default: "cc-droplets" @@ -250,26 +196,8 @@ properties: cc.buildpacks.connection_config: description: "Storage CLI connection configuration" cc.buildpacks.blobstore_type: - description: "The type of blobstore backing to use. Valid values: ['webdav', 'storage-cli']" + description: "The type of blobstore backing to use. Valid values: ['storage-cli']" default: "storage-cli" - cc.buildpacks.webdav_config.blobstore_timeout: - description: "The timeout in seconds for requests to the blobstore" - default: 5 - cc.buildpacks.webdav_config.public_endpoint: - description: "The location of the webdav server eg: https://blobstore.com" - default: "" - cc.buildpacks.webdav_config.private_endpoint: - description: "The location of the webdav server eg: https://blobstore.internal" - default: "https://blobstore.service.cf.internal:4443" - cc.buildpacks.webdav_config.username: - description: "The basic auth user that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.password: - description: "The basic auth password that CC uses to connect to the admin endpoint on webdav" - default: "" - cc.buildpacks.webdav_config.ca_cert: - description: "The CA certificate to use when communicating with webdav" - default: "" cc.buildpacks.buildpack_directory_key: description: "Directory (bucket) used store buildpacks. It does not have be pre-created." default: "cc-buildpacks" diff --git a/jobs/cloud_controller_worker/templates/buildpacks_ca_cert.pem.erb b/jobs/cloud_controller_worker/templates/buildpacks_ca_cert.pem.erb deleted file mode 120000 index 9c9bc307ac..0000000000 --- a/jobs/cloud_controller_worker/templates/buildpacks_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/buildpacks_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb index 01f66b3467..ea31b5c424 100644 --- a/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb +++ b/jobs/cloud_controller_worker/templates/cloud_controller_ng.yml.erb @@ -189,15 +189,6 @@ storage_cli_optional_flags: <%= p("cc.storage_cli_optional_flags") %> resource_pool: blobstore_type: <%= p("cc.resource_pool.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.resource_pool.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.resource_pool.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.resource_pool.webdav_config.private_endpoint") %> - username: <%= p("cc.resource_pool.webdav_config.username") %> - password: <%= yaml_escape(p("cc.resource_pool.webdav_config.password")) %> - <% if p("cc.resource_pool.webdav_config.ca_cert") != "" && p("cc.resource_pool.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_worker/config/certs/resource_pool_ca_cert.pem" - <% end %> minimum_size: <%= p("cc.resource_pool.minimum_size") %> maximum_size: <%= p("cc.resource_pool.maximum_size") %> resource_directory_key: <%= p("cc.resource_pool.resource_directory_key") %> @@ -205,43 +196,16 @@ resource_pool: packages: blobstore_type: <%= p("cc.packages.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.packages.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.packages.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.packages.webdav_config.private_endpoint") %> - username: <%= p("cc.packages.webdav_config.username") %> - password: <%= yaml_escape(p("cc.packages.webdav_config.password")) %> - <% if p("cc.packages.webdav_config.ca_cert") != "" && p("cc.packages.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_worker/config/certs/packages_ca_cert.pem" - <% end %> app_package_directory_key: <%= p("cc.packages.app_package_directory_key") %> max_package_size: <%= p("cc.packages.max_package_size") %> max_valid_packages_stored: <%= link("cloud_controller_internal").p("cc.packages.max_valid_packages_stored") %> droplets: blobstore_type: <%= p("cc.droplets.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.droplets.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.droplets.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.droplets.webdav_config.private_endpoint") %> - username: <%= p("cc.droplets.webdav_config.username") %> - password: <%= yaml_escape(p("cc.droplets.webdav_config.password")) %> - <% if p("cc.droplets.webdav_config.ca_cert") != "" && p("cc.droplets.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_worker/config/certs/droplets_ca_cert.pem" - <% end %> droplet_directory_key: <%= p("cc.droplets.droplet_directory_key") %> buildpacks: blobstore_type: <%= p("cc.buildpacks.blobstore_type") %> - webdav_config: - blobstore_timeout: <%= p("cc.buildpacks.webdav_config.blobstore_timeout") %> - public_endpoint: <%= p("cc.buildpacks.webdav_config.public_endpoint") %> - private_endpoint: <%= p("cc.buildpacks.webdav_config.private_endpoint") %> - username: <%= p("cc.buildpacks.webdav_config.username") %> - password: <%= yaml_escape(p("cc.buildpacks.webdav_config.password")) %> - <% if p("cc.buildpacks.webdav_config.ca_cert") != "" && p("cc.buildpacks.webdav_config.ca_cert") != nil %> - ca_cert_path: "/var/vcap/jobs/cloud_controller_worker/config/certs/buildpacks_ca_cert.pem" - <% end %> buildpack_directory_key: <%= p("cc.buildpacks.buildpack_directory_key") %> db_encryption_key: <%= p("cc.db_encryption_key") %> diff --git a/jobs/cloud_controller_worker/templates/droplets_ca_cert.pem.erb b/jobs/cloud_controller_worker/templates/droplets_ca_cert.pem.erb deleted file mode 120000 index d19041ffde..0000000000 --- a/jobs/cloud_controller_worker/templates/droplets_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/droplets_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/packages_ca_cert.pem.erb b/jobs/cloud_controller_worker/templates/packages_ca_cert.pem.erb deleted file mode 120000 index 1d24977063..0000000000 --- a/jobs/cloud_controller_worker/templates/packages_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/packages_ca_cert.pem.erb \ No newline at end of file diff --git a/jobs/cloud_controller_worker/templates/resource_pool_ca_cert.pem.erb b/jobs/cloud_controller_worker/templates/resource_pool_ca_cert.pem.erb deleted file mode 120000 index 25569fb402..0000000000 --- a/jobs/cloud_controller_worker/templates/resource_pool_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -../../../shared_job_templates/resource_pool_ca_cert.pem.erb \ No newline at end of file diff --git a/packages/blobstore_url_signer/packaging b/packages/blobstore_url_signer/packaging deleted file mode 100644 index 9e2bec4932..0000000000 --- a/packages/blobstore_url_signer/packaging +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -eu - -mkdir -p $BOSH_INSTALL_TARGET/bin - -source /var/vcap/packages/golang-1-linux/bosh/compile.env - -pushd github.com/cloudfoundry/blobstore_url_signer - go build -buildvcs=false -mod=vendor -o "${BOSH_INSTALL_TARGET}/bin/blobstore_url_signer" . -popd diff --git a/packages/blobstore_url_signer/spec b/packages/blobstore_url_signer/spec deleted file mode 100644 index 02da91ef05..0000000000 --- a/packages/blobstore_url_signer/spec +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: blobstore_url_signer - -dependencies: - - golang-1-linux - -files: - - github.com/cloudfoundry/blobstore_url_signer/**/* diff --git a/scripts/sync_submodule_config b/scripts/sync_submodule_config index 278f33656e..7bf52117ff 100755 --- a/scripts/sync_submodule_config +++ b/scripts/sync_submodule_config @@ -22,4 +22,4 @@ go get code.cloudfoundry.org/locket/cmd/... } > /tmp/packages -cat /tmp/packages | xargs gosub sync -i src/cloud_controller_ng -i src/github.com/cloudfoundry/blobstore_url_signer +cat /tmp/packages | xargs gosub sync -i src/cloud_controller_ng diff --git a/shared_job_templates/blobstore_waiter.sh.erb b/shared_job_templates/blobstore_waiter.sh.erb index 0c75b18e08..76fe62bbaf 100644 --- a/shared_job_templates/blobstore_waiter.sh.erb +++ b/shared_job_templates/blobstore_waiter.sh.erb @@ -27,10 +27,13 @@ function wait_for_blobstore { max_attempts=30 sleep_length=1 - <% if p('cc.buildpacks.blobstore_type') == 'webdav' %> + <% endpoint = (p('cc.buildpacks.connection_config', nil) || {}).fetch('private_endpoint', '').to_s %> + <% internal_hostname = p('blobstore.internal_hostname', 'blobstore.service.cf.internal').to_s %> + <% internal_singleton_blobstore = endpoint.match?(%r{\Ahttps?://#{Regexp.escape(internal_hostname)}(?::\d+)?(?:/.*|\z)}) %> + <% if p('cc.buildpacks.blobstore_provider', nil) == 'dav' && internal_singleton_blobstore %> echo "Checking for blobstore availability" - try_to_curl "<%= p('cc.buildpacks.webdav_config.private_endpoint') %>" $max_attempts $sleep_length + try_to_curl "<%= endpoint %>" $max_attempts $sleep_length echo "Blobstore is available" <% end %> diff --git a/shared_job_templates/buildpacks_ca_cert.pem.erb b/shared_job_templates/buildpacks_ca_cert.pem.erb deleted file mode 100644 index 3c650c76f8..0000000000 --- a/shared_job_templates/buildpacks_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('cc.buildpacks.webdav_config.ca_cert') %> diff --git a/shared_job_templates/droplets_ca_cert.pem.erb b/shared_job_templates/droplets_ca_cert.pem.erb deleted file mode 100644 index 8cb15668a7..0000000000 --- a/shared_job_templates/droplets_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('cc.droplets.webdav_config.ca_cert') %> diff --git a/shared_job_templates/packages_ca_cert.pem.erb b/shared_job_templates/packages_ca_cert.pem.erb deleted file mode 100644 index a0990d4ed1..0000000000 --- a/shared_job_templates/packages_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('cc.packages.webdav_config.ca_cert') %> diff --git a/shared_job_templates/resource_pool_ca_cert.pem.erb b/shared_job_templates/resource_pool_ca_cert.pem.erb deleted file mode 100644 index 898c4b6c18..0000000000 --- a/shared_job_templates/resource_pool_ca_cert.pem.erb +++ /dev/null @@ -1 +0,0 @@ -<%= p('cc.resource_pool.webdav_config.ca_cert') %> diff --git a/spec/blobstore/blobstore_spec.rb b/spec/blobstore/blobstore_spec.rb new file mode 100644 index 0000000000..ed7806e28c --- /dev/null +++ b/spec/blobstore/blobstore_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'rspec' +require 'bosh/template/test' + +module Bosh + module Template + module Test + describe 'blobstore job template rendering' do + let(:release_path) { File.join(File.dirname(__FILE__), '../..') } + let(:release) { ReleaseDir.new(release_path) } + let(:job) { release.job('blobstore') } + let(:template) { job.template('config/sites/blobstore.conf') } + + let(:manifest_properties) do + { + 'system_domain' => 'example.com', + 'blobstore' => { + 'secure_link' => { + 'secret' => 'super-secret' + } + } + } + end + + it 'uses the default internal singleton hostname' do + rendered_template = template.render(manifest_properties) + + expect(rendered_template).to include('server_name blobstore.service.cf.internal;') + end + + it 'uses the configured internal singleton hostname' do + manifest_properties['blobstore']['internal_hostname'] = 'blobstore.service.custom.internal' + + rendered_template = template.render(manifest_properties) + + expect(rendered_template).to include('server_name blobstore.service.custom.internal;') + end + end + end + end +end diff --git a/spec/cc_deployment_updater/cc_deployment_updater_spec.rb b/spec/cc_deployment_updater/cc_deployment_updater_spec.rb index ad61a5820c..f227affcad 100644 --- a/spec/cc_deployment_updater/cc_deployment_updater_spec.rb +++ b/spec/cc_deployment_updater/cc_deployment_updater_spec.rb @@ -49,15 +49,6 @@ module Test } end - let(:webdav_config) do - { 'blobstore_timeout' => 5, - 'ca_cert' => '((service_cf_internal_ca.certificate))', - 'password' => '((blobstore_admin_users_password))', - 'private_endpoint' => 'https://blobstore.service.cf.internal:4443', - 'public_endpoint' => 'https://blobstore.brook-sentry.capi.land', - 'username' => 'blobstore-user' } - end - let(:properties) do { 'router' => { 'route_services_secret' => '((router_route_services_secret))' }, @@ -81,25 +72,25 @@ module Test 'minimum_size' => 65_536, 'maximum_size' => 536_870_912, 'resource_directory_key' => 'cc-resources', - 'blobstore_type' => 'webdav', - 'webdav_config' => webdav_config + 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'packages' => { 'min_package_size' => 65_536, 'max_package_size' => 536_870_912, 'app_package_directory_key' => 'cc-packages', - 'blobstore_type' => 'webdav', - 'webdav_config' => webdav_config + 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'droplets' => { 'droplet_directory_key' => 'cc-droplets', - 'blobstore_type' => 'webdav', - 'webdav_config' => webdav_config + 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'buildpacks' => { 'buildpack_directory_key' => 'cc-buildpacks', - 'blobstore_type' => 'webdav', - 'webdav_config' => webdav_config + 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'credential_references' => { diff --git a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb index b1d749ef75..41f83daac7 100644 --- a/spec/cloud_controller_ng/cloud_controller_ng_spec.rb +++ b/spec/cloud_controller_ng/cloud_controller_ng_spec.rb @@ -23,14 +23,8 @@ module Test '((diego_ssh_proxy_host_key.public_key_fingerprint))' }, 'cc' => { 'buildpacks' => - { 'blobstore_type' => 'webdav', - 'webdav_config' => - { 'blobstore_timeout' => 5, - 'ca_cert' => '((service_cf_internal_ca.certificate))', - 'password' => '((blobstore_admin_users_password))', - 'private_endpoint' => 'https://blobstore.service.cf.internal:4443', - 'public_endpoint' => 'https://blobstore.brook-sentry.capi.land', - 'username' => 'blobstore-user' } }, + { 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'diego' => { 'file_server_url' => 'http://somewhere' @@ -45,14 +39,8 @@ module Test %w[public_networks dns load_balancer], 'default_staging_security_groups' => %w[public_networks dns], 'droplets' => - { 'blobstore_type' => 'webdav', - 'webdav_config' => - { 'blobstore_timeout' => 5, - 'ca_cert' => '((service_cf_internal_ca.certificate))', - 'password' => '((blobstore_admin_users_password))', - 'private_endpoint' => 'https://blobstore.service.cf.internal:4443', - 'public_endpoint' => 'https://blobstore.brook-sentry.capi.land', - 'username' => 'blobstore-user' } }, + { 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'experimental' => {}, 'install_buildpacks' => [{ 'name' => 'staticfile_buildpack', 'package' => 'staticfile-buildpack' }, @@ -69,24 +57,12 @@ module Test 'private_key' => '((cc_tls.private_key))', 'public_cert' => '((cc_tls.certificate))' }, 'packages' => - { 'blobstore_type' => 'webdav', - 'webdav_config' => - { 'blobstore_timeout' => 5, - 'ca_cert' => '((service_cf_internal_ca.certificate))', - 'password' => '((blobstore_admin_users_password))', - 'private_endpoint' => 'https://blobstore.service.cf.internal:4443', - 'public_endpoint' => 'https://blobstore.brook-sentry.capi.land', - 'username' => 'blobstore-user' } }, + { 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'rate_limiter' => {}, 'resource_pool' => - { 'blobstore_type' => 'webdav', - 'webdav_config' => - { 'blobstore_timeout' => 5, - 'ca_cert' => '((service_cf_internal_ca.certificate))', - 'password' => '((blobstore_admin_users_password))', - 'private_endpoint' => 'https://blobstore.service.cf.internal:4443', - 'public_endpoint' => 'https://blobstore.brook-sentry.capi.land', - 'username' => 'blobstore-user' } }, + { 'blobstore_type' => 'storage-cli', + 'blobstore_provider' => 'azurebs' }, 'security_group_definitions' => [{ 'name' => 'public_networks', 'rules' => @@ -195,6 +171,56 @@ module Test end end + describe 'bin/blobstore_waiter.sh' do + let(:template) { job.template('bin/blobstore_waiter.sh') } + + it 'does not wait for non-DAV blobstores' do + rendered_template = template.render(merged_manifest_properties, consumes: links) + + expect(rendered_template).not_to include('Checking for blobstore availability') + expect(rendered_template).not_to include('blobstore.service.cf.internal') + end + + it 'waits for the internal singleton DAV blobstore endpoint' do + merged_manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + merged_manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://blobstore.service.cf.internal:4443' + } + + rendered_template = template.render(merged_manifest_properties, consumes: links) + + expect(rendered_template).to include('Checking for blobstore availability') + expect(rendered_template).to include('try_to_curl "https://blobstore.service.cf.internal:4443" $max_attempts $sleep_length') + end + + it 'waits for the configured internal singleton DAV blobstore hostname' do + merged_manifest_properties['blobstore'] = { + 'internal_hostname' => 'blobstore.service.custom.internal' + } + merged_manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + merged_manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://blobstore.service.custom.internal:4443' + } + + rendered_template = template.render(merged_manifest_properties, consumes: links) + + expect(rendered_template).to include('Checking for blobstore availability') + expect(rendered_template).to include('try_to_curl "https://blobstore.service.custom.internal:4443" $max_attempts $sleep_length') + end + + it 'does not wait for an external DAV blobstore endpoint' do + merged_manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + merged_manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://external-blobstore.example.com:4443' + } + + rendered_template = template.render(merged_manifest_properties, consumes: links) + + expect(rendered_template).not_to include('Checking for blobstore availability') + expect(rendered_template).not_to include('external-blobstore.example.com:4443') + end + end + describe 'database_encryption block' do context 'when the database_encryption block is not present' do before do @@ -944,6 +970,17 @@ module Test expect(yaml['storage_cli_config_file_resource_pool']).to eq('/var/vcap/jobs/cloud_controller_ng/config/storage_cli_config_resource_pool.json') end end + + describe 'webdav_config removal' do + let(:template) { job.template('config/cloud_controller_ng.yml') } + + it 'does not render webdav_config for any blobstore scope' do + yaml = YAML.safe_load(template.render(merged_manifest_properties, consumes: links)) + %w[packages droplets buildpacks resource_pool].each do |scope| + expect(yaml[scope]).not_to have_key('webdav_config') + end + end + end end end end diff --git a/spec/cloud_controller_worker/cloud_controller_worker_spec.rb b/spec/cloud_controller_worker/cloud_controller_worker_spec.rb index 7927ea28ab..7de3da9dd2 100644 --- a/spec/cloud_controller_worker/cloud_controller_worker_spec.rb +++ b/spec/cloud_controller_worker/cloud_controller_worker_spec.rb @@ -376,6 +376,56 @@ module Test expect(template_hash['default_app_lifecycle']).to eq('cnb') end end + + describe 'bin/blobstore_waiter.sh' do + let(:template) { job.template('bin/blobstore_waiter.sh') } + + it 'does not wait for non-DAV blobstores' do + rendered_template = template.render(manifest_properties, consumes: links) + + expect(rendered_template).not_to include('Checking for blobstore availability') + expect(rendered_template).not_to include('blobstore.service.cf.internal') + end + + it 'waits for the internal singleton DAV blobstore endpoint' do + manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://blobstore.service.cf.internal:4443' + } + + rendered_template = template.render(manifest_properties, consumes: links) + + expect(rendered_template).to include('Checking for blobstore availability') + expect(rendered_template).to include('try_to_curl "https://blobstore.service.cf.internal:4443" $max_attempts $sleep_length') + end + + it 'waits for the configured internal singleton DAV blobstore hostname' do + manifest_properties['blobstore'] = { + 'internal_hostname' => 'blobstore.service.custom.internal' + } + manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://blobstore.service.custom.internal:4443' + } + + rendered_template = template.render(manifest_properties, consumes: links) + + expect(rendered_template).to include('Checking for blobstore availability') + expect(rendered_template).to include('try_to_curl "https://blobstore.service.custom.internal:4443" $max_attempts $sleep_length') + end + + it 'does not wait for an external DAV blobstore endpoint' do + manifest_properties['cc']['buildpacks']['blobstore_provider'] = 'dav' + manifest_properties['cc']['buildpacks']['connection_config'] = { + 'private_endpoint' => 'https://external-blobstore.example.com:4443' + } + + rendered_template = template.render(manifest_properties, consumes: links) + + expect(rendered_template).not_to include('Checking for blobstore availability') + expect(rendered_template).not_to include('external-blobstore.example.com:4443') + end + end end end end diff --git a/src/github.com/cloudfoundry/blobstore_url_signer b/src/github.com/cloudfoundry/blobstore_url_signer deleted file mode 160000 index 21272a29b6..0000000000 --- a/src/github.com/cloudfoundry/blobstore_url_signer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 21272a29b631a86ce8756a0efe1568e499e4b19d