diff --git a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
index 7c7f5e48d64..a8634a75ae3 100755
--- a/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
+++ b/systemvm/debian/opt/cloud/bin/cs/CsAddress.py
@@ -668,8 +668,12 @@ class CsIP:
                 logging.info("Not making dns publicly available")
 
             if self.config.has_metadata():
-                app = CsApache(self)
-                app.setup()
+                if method == "add":
+                    app = CsApache(self)
+                    app.setup()
+                elif method == "delete":
+                    app = CsApache(self)
+                    app.remove()
 
                 # If redundant then this is dealt with
                 # by the primary backup functions
diff --git a/systemvm/debian/opt/cloud/bin/cs/CsApp.py b/systemvm/debian/opt/cloud/bin/cs/CsApp.py
index d8b3223f017..123171a09c0 100755
--- a/systemvm/debian/opt/cloud/bin/cs/CsApp.py
+++ b/systemvm/debian/opt/cloud/bin/cs/CsApp.py
@@ -34,7 +34,7 @@ class CsApache(CsApp):
     """ Set up Apache """
 
     def remove(self):
-        file = "/etc/apache2/sites-enabled/vhost-%s.conf" % self.dev
+        file = "/etc/apache2/sites-enabled/vhost-%s.conf" % self.ip
         if os.path.isfile(file):
             os.remove(file)
             CsHelper.service("apache2", "restart")