Disable logs to stdout and capture remaining stdout logs to a file (#8880)

* Log stdout to a file

* Add logrotation

agent/conf/cloudstack-agent.logrotate.in

@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log {
+/var/log/cloudstack/agent/security_group.log /var/log/cloudstack/agent/resizevolume.log /var/log/cloudstack/agent/rolling-maintenance.log /var/log/cloudstack/agent/agent.out /var/log/cloudstack/agent/agent.err {
     copytruncate
     daily
     rotate 5
     compress
     missingok
     size 10M
+    dateext
+    dateformat -%Y-%m-%d
 }

agent/conf/log4j-cloud.xml.in

@@ -38,7 +38,7 @@ under the License.
       <!-- ============================== -->
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
-          <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+          <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
           <PatternLayout pattern="%-5p [%c{3}] (%t:%x) (logid:%X{logcontextid}) %m%ex%n"/>
       </Console>
    </Appenders>

client/conf/log4j-cloud.xml.in

@@ -69,7 +69,7 @@ under the License.
       <!-- ============================== -->
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
-         <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+         <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
          <PatternLayout pattern="%-5p [%c{1.}] (%t:%x) (logid:%X{logcontextid}) %m%ex{filters(${filters})}%n"/>
       </Console>
 

debian/rules

@@ -103,6 +103,8 @@ override_dh_auto_install:
 	install -m0644 packaging/systemd/$(PACKAGE)-management.service debian/$(PACKAGE)-management/lib/systemd/system/$(PACKAGE)-management.service
 	install -m0644 packaging/systemd/$(PACKAGE)-management.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-management
 
+	install -D -m0644 server/target/conf/cloudstack-management.logrotate $(DESTDIR)/$(SYSCONFDIR)/logrotate.d/cloudstack-management
+
 	# cloudstack-ui
 	mkdir $(DESTDIR)/$(SYSCONFDIR)/$(PACKAGE)/ui
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-ui
@@ -159,6 +161,8 @@ override_dh_auto_install:
 	install -m0644 packaging/systemd/$(PACKAGE)-usage.service debian/$(PACKAGE)-usage/lib/systemd/system/$(PACKAGE)-usage.service
 	install -m0644 packaging/systemd/$(PACKAGE)-usage.default $(DESTDIR)/$(SYSCONFDIR)/default/$(PACKAGE)-usage
 
+	install -D -m0644 usage/target/transformed/cloudstack-usage.logrotate $(DESTDIR)/$(SYSCONFDIR)/logrotate.d/cloudstack-usage
+
 	# cloudstack-marvin
 	mkdir -p $(DESTDIR)/usr/share/$(PACKAGE)-marvin
 	cp tools/marvin/dist/Marvin-*.tar.gz $(DESTDIR)/usr/share/$(PACKAGE)-marvin/

packaging/centos7/cloud.spec

@@ -314,6 +314,7 @@ install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_s
 install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
 touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
 #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
+install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management
 
 # SystemVM template
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
@@ -378,6 +379,7 @@ cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/
 install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service
 install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/
+install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage
 
 # CLI
 cp -r cloud-cli/cloudtool ${RPM_BUILD_ROOT}%{python_sitearch}/
@@ -623,6 +625,7 @@ pip3 install --upgrade urllib3
 %{_defaultdocdir}/%{name}-management-%{version}/LICENSE
 %{_defaultdocdir}/%{name}-management-%{version}/NOTICE
 #%attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-catalina
+%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management
 %{_datadir}/%{name}-management/setup/wheel/*.whl
 
 %files agent
@@ -670,6 +673,7 @@ pip3 install --upgrade urllib3
 %files usage
 %attr(0644,root,root) %{_unitdir}/%{name}-usage.service
 %config(noreplace) %{_sysconfdir}/default/%{name}-usage
+%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage
 %attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar
 %attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar
 %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage

packaging/centos8/cloud.spec

@@ -296,6 +296,7 @@ install -D packaging/systemd/cloudstack-management.default ${RPM_BUILD_ROOT}%{_s
 install -D server/target/conf/cloudstack-sudoers ${RPM_BUILD_ROOT}%{_sysconfdir}/sudoers.d/%{name}-management
 touch ${RPM_BUILD_ROOT}%{_localstatedir}/run/%{name}-management.pid
 #install -D server/target/conf/cloudstack-catalina.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-catalina
+install -D server/target/conf/cloudstack-management.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-management
 
 # SystemVM template
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-management/templates/systemvm
@@ -360,6 +361,7 @@ cp client/target/lib/mysql*jar ${RPM_BUILD_ROOT}%{_datadir}/%{name}-usage/lib/
 install -D packaging/systemd/cloudstack-usage.service ${RPM_BUILD_ROOT}%{_unitdir}/%{name}-usage.service
 install -D packaging/systemd/cloudstack-usage.default ${RPM_BUILD_ROOT}%{_sysconfdir}/default/%{name}-usage
 mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/log/%{name}/usage/
+install -D usage/target/transformed/cloudstack-usage.logrotate ${RPM_BUILD_ROOT}%{_sysconfdir}/logrotate.d/%{name}-usage
 
 # Marvin
 mkdir -p ${RPM_BUILD_ROOT}%{_datadir}/%{name}-marvin
@@ -577,6 +579,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %config(noreplace) %{_sysconfdir}/%{name}/management/log4j2.xml
 %config(noreplace) %{_sysconfdir}/%{name}/management/environment.properties
 %config(noreplace) %{_sysconfdir}/%{name}/management/java.security.ciphers
+%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-management
 %attr(0644,root,root) %{_unitdir}/%{name}-management.service
 %attr(0755,cloud,cloud) %{_localstatedir}/run/%{name}-management.pid
 %attr(0755,root,root) %{_bindir}/%{name}-setup-management
@@ -648,6 +651,7 @@ pip3 install --upgrade /usr/share/cloudstack-marvin/Marvin-*.tar.gz
 %files usage
 %attr(0644,root,root) %{_unitdir}/%{name}-usage.service
 %config(noreplace) %{_sysconfdir}/default/%{name}-usage
+%config(noreplace) %attr(0644,root,root) %{_sysconfdir}/logrotate.d/%{name}-usage
 %attr(0644,root,root) %{_datadir}/%{name}-usage/*.jar
 %attr(0644,root,root) %{_datadir}/%{name}-usage/lib/*.jar
 %dir %attr(0770,root,cloud) %{_localstatedir}/log/%{name}/usage

packaging/systemd/cloudstack-agent.service

@@ -31,6 +31,8 @@ EnvironmentFile=/etc/default/cloudstack-agent
 ExecStart=/usr/bin/java $JAVA_OPTS $JAVA_DEBUG -cp $CLASSPATH $JAVA_CLASS
 Restart=always
 RestartSec=10s
+StandardOutput=append:/var/log/cloudstack/agent/agent.out
+StandardError=append:/var/log/cloudstack/agent/agent.err
 
 [Install]
 WantedBy=multi-user.target

packaging/systemd/cloudstack-management.service

@@ -35,6 +35,8 @@ EnvironmentFile=/etc/default/cloudstack-management
 WorkingDirectory=/var/log/cloudstack/management
 PIDFile=/var/run/cloudstack-management.pid
 ExecStart=/usr/bin/java $JAVA_DEBUG $JAVA_OPTS -cp $CLASSPATH $BOOTSTRAP_CLASS
+StandardOutput=append:/var/log/cloudstack/management/management-server.out
+StandardError=append:/var/log/cloudstack/management/management-server.err
 
 [Install]
 WantedBy=multi-user.target

packaging/systemd/cloudstack-usage.service

@@ -35,6 +35,8 @@ Environment=JAVA_PID=$$
 ExecStart=/bin/sh -ec '/usr/bin/java -Dpid=${JAVA_PID} $JAVA_OPTS $JAVA_DEBUG -cp $CLASSPATH $JAVA_CLASS'
 Restart=always
 RestartSec=10s
+StandardOutput=append:/var/log/cloudstack/usage/usage.out
+StandardError=append:/var/log/cloudstack/usage/usage.err
 
 [Install]
 WantedBy=multi-user.target

server/conf/cloudstack-management.logrotate.in

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+/var/log/cloudstack/management/management-server.out /var/log/cloudstack/management/management-server.err {
+    su root root
+    copytruncate
+    daily
+    rotate 5
+    compress
+    missingok
+    size 10M
+    dateext
+    dateformat -%Y-%m-%d
+}

server/conf/log4j-cloud.xml.in

@@ -67,7 +67,7 @@ under the License.
       <!-- ============================== -->
 
       <Console name="CONSOLE" target="SYSTEM_OUT">
-         <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+         <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
          <PatternLayout pattern="%-5p [%c{1.}] (%t:%x) %m%ex%n"/>
       </Console>
    </Appenders>

usage/conf/cloudstack-usage.logrotate.in

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+/var/log/cloudstack/usage/usage.out /var/log/cloudstack/usage/usage.err {
+    su root root
+    copytruncate
+    daily
+    rotate 5
+    compress
+    missingok
+    size 10M
+    dateext
+    dateformat -%Y-%m-%d
+}

usage/conf/log4j-cloud_usage.xml.in

@@ -25,7 +25,7 @@ under the License.
        <!-- ============================== -->
 
        <Console name="CONSOLE" target="SYSTEM_OUT">
-          <ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
+          <ThresholdFilter level="OFF" onMatch="ACCEPT" onMismatch="DENY"/>
           <PatternLayout pattern="%d{ABSOLUTE}{GMT} %5p %c{1}:%L - %m%ex%n"/>
        </Console>